{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Handwritten Digit Recognition\n",
    "\n",
    "This tutorial guides you through a classic computer vision application: identify hand written digits with neural networks. \n",
    "\n",
    "## Load data\n",
    "\n",
    "We first fetch the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset, which is a commonly used dataset for handwritten digit recognition. Each image in this dataset has been resized into 28x28 with grayscale value between 0 and 254. The following codes download and load the images and the according labels into `numpy`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import urllib\n",
    "import gzip\n",
    "import struct\n",
    "def download_data(url, force_download=True): \n",
    "    fname = url.split(\"/\")[-1]\n",
    "    if force_download or not os.path.exists(fname):\n",
    "        urllib.urlretrieve(url, fname)\n",
    "    return fname\n",
    "\n",
    "def read_data(label_url, image_url):\n",
    "    with gzip.open(download_data(label_url)) as flbl:\n",
    "        magic, num = struct.unpack(\">II\", flbl.read(8))\n",
    "        label = np.fromstring(flbl.read(), dtype=np.int8)\n",
    "    with gzip.open(download_data(image_url), 'rb') as fimg:\n",
    "        magic, num, rows, cols = struct.unpack(\">IIII\", fimg.read(16))\n",
    "        image = np.fromstring(fimg.read(), dtype=np.uint8).reshape(len(label), rows, cols)\n",
    "    return (label, image)\n",
    "\n",
    "path='http://yann.lecun.com/exdb/mnist/'\n",
    "(train_lbl, train_img) = read_data(\n",
    "    path+'train-labels-idx1-ubyte.gz', path+'train-images-idx3-ubyte.gz')\n",
    "(val_lbl, val_img) = read_data(\n",
    "    path+'t10k-labels-idx1-ubyte.gz', path+'t10k-images-idx3-ubyte.gz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We plot the first 10 images and print their labels. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAABVCAYAAADQZjepAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3VlwVFd++PFv761uSa2W1K1933cLCQsBEiDZ2AaDbSA2\ntqcynknKlaRSleQhT0klD3nKUpVypVI1Sf0dL/HYjgsPzBgbGYyRABkQuyS07/u+tFotqdf/A3Wv\nkTFehbqFzqeKF2G3fre5y++e8zu/o/D5fAiCIAiCsPko/R2AIAiCIAj+IZIAQRAEQdikRBIgCIIg\nCJuUSAIEQRAEYZMSSYAgCIIgbFIiCRAEQRCETUokAYIgCIKwSYkkQBAEQRA2KZEECIIgCMImJZIA\nQRAEQdikRBIgCIIgCJuUSAIEQRAEYZMSSYAgCIIgbFIiCRAEQRCETUokAYIgCIKwSan9HQCAQqHw\n+TuG7+Pz+RQP+rtAj1/E7j8Pin8jxw6BH7+I3X82cvyPauzfRYwECIIgCMImJZIAQRAEQdikRBIg\nCIIgCJtUQNQECPcLDg7GYrGQmJiI2WwGYGZmhoGBASYnJ1lcXPRzhIKwOalUKnJycoiPj8dkMuF0\nOmlvb6e3t5elpSV/hyc8gnJyckhJSQHg5s2bjI6OrtlnP/JJgFqtRqVSAaBQKPD5fDidTjQaDWq1\nGoXibi2F1+vF4/Hgcrnw+fxb/6FUKklISKCiooLDhw9TVFSEz+fj9u3bHDt2jPPnz9PR0eHXGH8q\nhUKBWq1Go9HgdrsD4vv+Pmq1etV55PV6cTqdeL3egI99I1IoFKhUKhQKBQqFIuDOEb1ez0svvcRz\nzz1HTk4Oc3NzvPHGG7z99tsMDw8HVKzfJH2ncDeZkf58F5/Ph9vtxul0rkeIP4h0jmg0GlwuF263\n298hPRQKhQKNRsOhQ4f4xS9+gdfr5W//9m9FEvBtFAoFSqWSoKAglMqvZzm2bt1KQUEBWq2WyMhI\nxsbG+PDDD9m/fz/bt28nKSkJr9fLwMAAV65c4dNPP2VoaMhvx6HVaklKSuLll1/mueeeIyYmhpCQ\nEAC2bNlCdHQ0er1+wyYBcXFxbN26lcOHD1NbW8tnn33G6OhoQN84KyoqqKqqoqysDLVaTXNzMx9+\n+CGtra3Mzs76O7xHhlarRafTodfryc/PJzExEa1Wy+eff87w8DAej8ffIRIUFERcXByFhYUkJCSg\nVCoJCQkhLi6O+Ph4RkdHAyLOb6NSqbBarRgMBoxGIxkZGezYsYPHHnvsO6+/kZERzp8/z1tvvYXH\n4/H7tapUKuV/g3379nHy5ElOnTrl15gelvDwcI4ePUpVVRUhISF0dXWt+WjThksCNBoNBoMBpVJJ\naGgoVqsVi8WC0WgkKCiI6OhotFqt/N9nZWWRkpKCRqPBaDQyMjKCy+XiqaeeoqSkhMjISBwOByqV\nitu3b69KINZbSEgIqampHDhwgKeeeoqcnByUSiU+nw+v10toaCgmk4nc3Fzi4uKYnJx8KG9JwcHB\nREVFodVqmZ2dZWxsbM0+22KxUFhYSGFhIZ2dnQQFBckjNIEmODiYtLQ09u7dy759+8jNzUWlUqHV\navniiy9WnWfCT6PT6YiIiCA/P5/Y2FjCw8Pl68BqtaJUKtHpdNTW1tLc3OzvcIG7I0GAfM5K9xaD\nwSC/ZQeSsLAw4uLiSEtLIzs7G5PJRFBQEElJSRQUFJCenv6d19/09DQmk4m5uTkaGhoYHh7265u3\nTqejuLiYV155hdLSUtra2tDr9SwvL/stpochOTmZHTt2cPDgQbKzs3E6nbS2tmKz2db092yoJECj\n0RAVFUVGRgYajYa4uDhycnLIyckhIiKC0NBQUlJSCAoKWvX/SSf49PQ0Pp+PAwcOkJCQgMfjYWBg\ngOnpaZqammhsbGRhYWHdj0upVKLVaklLS6O6uppf/epXxMfHy8N0TqcTh8PB4uIikZGRJCcns3Pn\nTq5fv87o6Oia1wdERERQVlZGWFgYTU1Na5oEmM1mEhISWFpaCviLNiwsjF27dlFZWUlOTo6crEh/\n/E2tVqPX6zEYDOh0ulVTFm63G7vdjtvtls+vpaUllpaW/D6sq1Qq0Wg0hIaGEh0dTU5ODn/0R39E\nbm4u0dHR8sgX3L12Q0ND5Xl3t9vt1+/e6XQyNzfHyMgI8/PzmEwmv8XyQyiVSjIyMqisrKS6upqC\nggJMJpOcXEkvGd8lIiKCbdu2YTKZ+Nd//Vfm5uaYn59fpyNYTaFQoNfrKS4u5siRI6ysrGA2mzGZ\nTKysrATEdblW8vPzefnll3n88cfR6XS0tLTQ0NDA1NTUmv6eDZUEJCQksH//fv78z/8cjUaDTqeT\n/6jVavlm9yBXrlzh6tWrLC4uotFocDqdzM7OMjs7S39/Px0dHev+YFKpVAQFBZGZmcmhQ4d46aWX\niI6OXjVP53A4aGxs5OLFi7z66qts376d+Ph4Tp06xccff0xDQ8OaxmQ2mykuLiY+Pp6lpSUuXLiw\nJp+rUCgIDQ2V3/j0ev2afO7DotFosFqtBAcH+3WE6EGsViv5+flUVlaSlZVFXFwcoaGhqFQqxsbG\nOH36NBMTE+h0OrKzs7l8+TLXr1+nvb3dr3EHBweTnJzMwYMHKS8vJysrC7PZLE/lLS8v43K5UKlU\nGI1GMjMzycvLIyEhgcHBQVwul99i93g8LC0tMT097ZcXhh9Lo9Fw8OBB9u3bR1paGnq9flWN1A9l\nMpkoKioiLy+Prq4uvyUB91IoFOh0OsLCwjCbzUxOTj5SSUB4eDhpaWnodDoWFhbo6OigpqZmcycB\ny8vLKBQK4uPj0ev1qNXfHr7D4cDhcOByuQgODsZgMODz+bh58yaffPKJPJTlcrlYXl5meXmZxcVF\nv1Tcp6ens2/fPvLz8ykuLiYhIWFVwSIgX7jz8/NMTU1hsVhISkqisLBwzR7Q97JYLJSXl+PxeNb0\nQR0cHExCQgIZGRksLS0xPj4uj84EmpiYGLZs2UJ5eTlRUVF4PB6Wl5c5ffo0X3zxBZ2dnX5boaHR\naMjKyqKyspI9e/aQkZGB1WrFaDSysrKCSqUiLCwMo9HI4uIiarUaq9VKZGQkSqXSL0mAlKDv2LGD\nkpIScnNzyc/PJyEhgfDwcLkAsLu7mxMnTuByucjPz+f5558nODiYiIgIIiIiGB0d9WsSoFKp0Ov1\nmM1mjEaj3+L4oXw+HwsLCzidTnmExePx4HA46OvrY2lpadV0nF6vx2q1ylMGErVajcFgICgoKGCm\nwaQiR6lwN9AplUoSExPJy8sjOjqaU6dOMTExcd/UirT6JC8vj4iICFwuFw0NDXz++edMT0+v+fkf\n+N/cPebm5hgcHKSjo4OoqCjUajVOpxOLxYJer8fn8+Fyuejq6qK1tZWlpSWioqKIi4vDYrHQ0dHB\nzZs3/X0YMoPBQH5+Pr/85S9JSUkhODgYr9fLysoKdrsdr9dLZGSkPNLhcDjo6urCZDKRkJBAZGQk\nBoNhzeMymUxkZWXR19e3Zp+pVqvJy8ujqKiIqKgorl27xtjYGHNzc2v2O9aCSqUiJCSEkpIS9u7d\ny2OPPUZISAh2u53u7m4+/vhjzpw5w8LCgl/mRXU6HRaLhSeffJL9+/dTXl6Oz+djdnZWXqamUCiw\nWCwUFBQQHByMRqMB7ibHTU1N6x4z3H24SCN5zzzzDCkpKahUKnk42uPx0NPTw5kzZ/h//+//oVar\neeGFF3jhhRfkaQ+j0ej3OXe1Wi3XzNw7FaDT6QgJCSEoKIjFxUW5bsDfPB4Pt27dwmKxyA96t9vN\n/Pw8N27cwG63A19PmQYHB5OSkkJZWRmpqakolUo5QVtcXGR+fh6Hw+G34/k2UlF4oFOr1Wzbto1n\nn32W+Ph42tvb5e/0m//dzp07KSkpQafT0dvby7lz56irq3so95wNlQQ4HA7u3LnD22+/TXl5OQCT\nk5McOnSI+Ph4PB4Pc3NzHDt2jP/6r/9iaWmJuLg4SktLOXLkCNPT034+gtUSEhLIzc0lPT0dnU4H\n3L1ox8fHuX37Ni6Xi4MHD+L1epmenqalpQW73Y5OpyMhIWHVcp+1ttafq9Vq+eM//mOeeOIJ3G43\nIyMja17gshYMBgOFhYUcOXKEAwcOEBoaikKhYGxsjBMnTtDU1MTMzIxfRi+USiVWq5XHH3+co0eP\nkp+fj1KpZGxsjDNnznDy5EmuXbuGy+WiqKiIf/zHfyQnJ0fuMzE8POy3lS9ms5k9e/awfft20tLS\nVt20vV4vi4uLHDt2jLfffpuBgQFiY2P9Euf3kabvQkJCVo2SWa1W0tPTaW1tZXh4OGD6eHg8Hurq\n6mhsbOTdd9+Vf7a8vCwXFt9LrVYTEhLCP/3TP8kjrgB2u53Ozk6am5sZHBxc9+P4LtLUsL8TxO+j\n1Wo5dOgQ1dXVLCwskJmZydDQ0H1JgEajYc+ePRQXF+NwODh+/Djnzp1jYGDgoSSXGyoJgLvLVWpq\namhubpbfIGJjY9m+fTtGo5G6ujpu3brFzMwMHo+HwcFBHA4H/f39dHd3+zt84OtlRr/4xS/Yv3+/\nfALbbDY6Ojp46623WFlZISEhAb1ez82bN7l06RItLS309fWRk5PDwYMH5aH1hIQEhoaG1uTBFBUV\nRWxsrFw0tFYUCoX8prS8vExzczPj4+Nr9vlrJTQ0lN27d5Obm0tISAhKpZL5+Xna2tr45JNPGBwc\n9FsCUFpaSnV1Nc888wzp6ek4nU56enp49913aWhooLe3l+npaTIyMigoKCA+Ph6DwSD3v7h69SrX\nr19f99jhbhJQWVlJTEwMKpUKn8/HysoK8/Pz9PX18cknn/DFF1/IS+wsFgsWi8UvsX6XlZUVJicn\nuXXrFvHx8WRnZwOQnZ0tH8/Zs2fp7e31c6Rfczqdq2oYpNVG31ZIZzabycnJwWKxrBpiX15eZmJi\nQp5aCCQWi4W4uDhu3boVsMszJff2ZggKCrpvGiMkJIT09HR5KafT6ZSfcw9rdGnDJQF2u52uri75\nZmwwGOjp6SEnJwej0Yjb7cbtdssng1QfECjZqzQsumvXLp544gny8vLwer243W4GBwe5fPkyp06d\nkvsFdHd3c/PmTfr6+uRhuOnpaVQqFWazmaKiIkpKShgbG1uTuaLk5GRSU1PRaDRrllmrVCp5bbJO\np2Nubo6enh5mZmbW5PPXilqtJjw8nLKyMpKSklCpVHi9Xjo7O7l69Sqtra1+mY+OiIggKyuLffv2\nUV1dTUlJCdPT0zQ3N1NbWysnJ1Jsubm57N69G4vFgk6nY2Zmhjt37nDjxo01neL5MZaXlxkaGqK3\nt5eZmRmmp6eZnp5mfHyc7u5uPvvsM4aGhuTCXLPZLI9gBBKPx4PNZuPcuXMkJibKSYDVaiUjI4OU\nlJSAqxWQpkm/79yNjIwkJyeH6upqEhMTVxUnT05OUldXFxDFd1IS4/F4UCqVhIeHExMTE9AjAdIy\n2PDwcODuc2l2dnZVIbrU/2DPnj1ERUXh8/mYn59nYGDgoY6abrgkQLKysgLcfajOz8+zuLiIXq9n\n+/btXLlyhS+++CIgu0hZLBa2bNnC0aNHSU1Nxev1YrfbsdvtNDY2cv78eWZnZ7HZbHR2dn7nZ6lU\nKoqLixkeHubzzz9fkwdUZmYmmZmZq/oT/FxS/waz2YxOp8PlcjExMREwQ6bw9UhFQkIC+fn5REZG\nypXgX331FbW1tX55y1Cr1eTm5vL666+ze/duYmJicDgc3Lp1i//7v//j448/luegpbnRoqIiKisr\n0el0eDwe+vr6eOedd2hqavLbfO7g4CBvvPEGzz77LCEhIXJiK71dflNwcDDBwcF+iPT7uVwuampq\nyMvL48UXX/R3OD+LUqmUC+uKiorYu3cvTz31FImJifJIoM/no6enhzfffNPv16zUvVBKarRaLSaT\nicjIyIBNAhQKBZmZmfzpn/4pmZmZ+Hw+xsbGaG9vX/UiZDAY5HMqOjqahYUFOjs7OX/+PAMDAw8t\nvg2bBEgcDge/+93v0Ov1xMXFYTab5Ur7pqamgFqLrtFo2L17N6+88goFBQUYDAZaWlo4ceIEHR0d\njI2NMTk5KSc4P4TBYCA0NHTNYpSGYb1eL8PDw2tSR5GTk8Prr79OWloaXq8Xm83G4uKiX6u876VS\nqYiIiOC5557j6NGjcrY+NzfHlStXOH/+PC0tLX4p9tq7dy8HDx6kqqqKsLAw5ufn6ezs5J133qG+\nvh6Hw4HX60WlUmEymeTmL3q9HqVSSWNjo7yiYXJyct3jlzidTqampjh58iRqtRqbzcby8vIDh5aT\nkpJITExc5yh/vEBtdPVDZWVlUV5eTmVlJWlpacTGxhIREbFqZcDCwgI2m83vPRokS0tLTE1NMTo6\nSlxcnL/D+V4mk0nuzmgymbh69Srvv/8+g4ODq55PBw8e5PDhw2RmZqJWqzlx4gRvvPEGo6OjD/Xe\ns+GTALfbTU9PD7W1tURHR3PgwAGKi4t5+eWXyc7OZmxsjJGRETo6Ovz60NFoNMTGxvLYY4+xZcsW\nzGYz169fp6amhj/84Q+MjIywsrIiV+L+UFIf/rXKgvV6PXq9Hq/Xy9DQ0E96cKhUKkJDQ4mLiyMp\nKYmKigr27NlDREQEIyMjXLt27aEsdfmppHX027Zto7S0FL1ez+LiIj09PfzhD3+gsbFx3ddFSw11\nysvLqaioICoqSu7YduLECerr6xkZGcHn88lruB9//HFyc3MpKCjA5/MxPj5OXV0dp0+fZnh42K8V\n61INwPDw8Kqfx8bGkpycfN+y2Pz8fKxWKz6fj8XFRcbHxxkaGgrI0b1AZzQaiYmJITk5WV5JJcnM\nzKSkpITCwkJCQ0Pl5X8+n4/p6Wk6Ojro6Oigrq5O3i/D36TCxqWlpYCI50G0Wi1ms5mqqir27dtH\ndHQ0Y2NjNDQ0UFtby9zcHF6vF5PJRF5eHk8++SSPP/44RqORpqYmrly5wu3btx96nBs+CYC7w3M3\nbtzA7XaTlZVFTk4Or776KhUVFfT09HD16lXsdjuTk5MsLy/75cQJCgqiuLiYnJwcwsPDcbvdnD59\nmmPHjgVMO9R7eb1eJiYm7lvCJz2cpPW5Ug8DpVIpdyELCgoiJiaGrVu3sm3bNnk9uEqloru7my++\n+IKpqamAuKFrtVosFgvbtm0jOzsbo9Eod5K8dOkSp06d8ksBo9QoJzs7Wx5B6ezs5NNPP+Wtt97C\naDTKjYzi4uJ48cUXOXToEGazGbVazezsLC0tLXz55Zdcvnw5YG6W0sYvWq2W0NBQysvLqaqquq8Q\ntaCggLCwMNxuN93d3dy5c4eenp6AOQ5JILwZP4hSqUSv15OVlUVZWRmVlZXk5uauGjk0mUzf2vVw\nZWWF3t5ePvzwQxoaGujq6gqYpP2bpHtPIE0HSFOgRUVFvPbaa+zYsQOlUsnQ0BB9fX1MTU3JNQ2x\nsbEcOXKE8vJyLBaLXHPS2Ni4LrE+EkkA3B2yam9v57333uPIkSNUVFSQk5NDWloamZmZBAcH8+WX\nX9LU1OSXTV/CwsJ4+eWXKSoqwul0Mj4+TmNj4/fO+z+INAz5sJYJKhQKDAbDfc2CDAYDqampqNVq\noqOjycvLIzIyUl42lZubi9FoRK1Wo9PpWFxclGsLFAoFU1NTdHZ2BkyFcVJSEnv27OHIkSNkZmbi\ncrmYmprid7/7HR988MG3NvNYD1KypdVqUavVrKys8NFHH1FXV0d0dDTV1dVs3bqV3NxcIiMjsVqt\nmM1m+UE6OzvLl19+SW9vb8B813B3RCwiIoKMjAwOHz5MeXk5KSkpq85jn88ndw+02Wy88847nDlz\nJuASgEAXGhpKeno6f/VXf8W2bdvkLp33JlsP2kFwZmaGxsZGjh07xvz8fECdQ98UFBQUUPUjSqVS\n3tzo6NGjxMTEyH1sEhMTqa6uxuv1cuLECVZWVkhMTGTnzp1YLBbm5uZoamrik08+WZdRAHiEkgCv\n18vc3By1tbWsrKwwMjJCRUUF0dHRpKamcvDgQcLDw7FYLNTW1mKz2dYts42KipJHAcxmMzabjZs3\nb8pTAD+HzWZb06570uoKjUZDRUUFZrOZJ598Uv57g8Egv9UHBwdjtVoJCgqSt9dVKpWMjIwwOjpK\nf38/4+Pj5Obm8qtf/QqlUsns7CwDAwN+f6uQWhiXl5fzyiuvkJ6eTnBwsNwStre3l76+Pr/d/O6t\ngPZ4PKjVap544gmysrJQKpWkpaWRkJAgL+VSKpV4vV6USiVut5upqSnq6urWbOnozyEllFLyUlBQ\nICeQ925UFRwcLC/LBOTkKyIiQu53LxKBH05alRMdHU1MTMx9e6oAqxKve4WEhBAdHU1SUhIdHR1r\nvnPdWoqIiCA2NjYgGgYFBQWRn5/PwYMH5aZYarUat9vNysoKYWFhbNmyhYiICFJSUgBISUkhJSUF\nnU7HrVu3+M///E9aWlrWrYj3kUkC4G7xUVdXl9xZ0OFwUFhYSEpKCvn5+YSEhGAymXC73Vy9epWR\nkZF1iSsyMpLs7GzCw8NRq9XMzMxw8eLFHz3frlariYmJkQv3lpeX6ejooKmpac0q1/v7+2ltbSUi\nIoLU1FQSExNXJSoajYaQkBAWFhaw2+0sLS0xMzODzWZjdnaWoaEhBgYG6OnpoaWlRX5Awd25PLvd\nHhBLAzUaDdnZ2ezcuZOKigr550tLS7S0tDA4OOjXG5/079vd3U1/fz8pKSk8/fTTeL1eXC4Xdrud\nubk5Ojo6cDgcREZGEhcXR3BwMHa7naGhoYDY6lhKtlJTU9m5cycvv/wyWVlZ2O12bDYbLS0tzM7O\nMjU1RUFBAbm5uXLfDKk3fFlZGSMjI4yMjDA1NRVQb6X3Fgaq1WrMZnPAtNV1uVxyjwuz2SwPNbtc\nLjmZujcJkHrwm0wmQkJCiI+PJzc3l/Hxcb+fR9+0vLzM/Py8vLtqRESE36cDjEYjKSkp7N+/n6ef\nfprCwkK8Xi+Dg4NMTk7icDhIT08nJiaGuLg4cnNz0Wg06PV6tFotbrcbh8PB2NiYPBK4Hi9Lj1QS\nIJmenubSpUu0t7ezdetWnnnmGV599VWSk5Mxm83k5eXxd3/3d+uWBKhUKnQ6HSqVCo/HI3d4Gx0d\n/cGfoVQqMRqNvPjii1RUVOByuRgbG+PkyZMcO3ZszU6W48ePMzg4yOzs7H3zh5LJyUmuXr1Kc3Mz\nfX193L59W+6i5/V6V73B7t+/n7KyMrRabUCtCAgKCuLZZ59l69atq34+PT3NRx995Pc6DbfbzcLC\nAh999BFer5e//Mu/RKvV4nK5mJ2d5fbt29TV1XHhwgXm5uZ46aWX+OUvf4nBYGBwcJC2traAqLlQ\nqVRkZmby7LPP8hd/8ReEhobS29vL2bNnuXXrFjdv3qSzs5Pg4GD+5m/+huTkZLRarbzc0Wg0smvX\nLlwuFzMzM5w7d05uBBYopEQgJCSEwsLCgOlvsLCwwJ07d/j3f/936uvr2bZtG9euXWNiYuJbE6md\nO3dSXV1NWVmZ3ME0UE1MTNDS0kJ2dnbAJF3SNvCvvfYa0dHReDwenE4nx48f58KFC6ysrPBnf/Zn\nbN26lYiICHnPDCl5UavV7N69m8zMTP7+7/+e2tradVnR80gmAT6fT16S1NDQgNFo5PDhwxiNRnmv\n8qKiIrq6utali+C9GarP55M3z/mhb5oqlYr09HS5X3xqaipTU1PU1NTI2x+v1ZCvtGf1//zP/2Ay\nmR54gU1NTTE7O4vdbmd2dvaBb2dpaWmkp6fLhW3+alt7L2lvhIKCglXtaWdmZuju7qajoyNg3nw6\nOzv57W9/y40bN+ThcKfTyczMjLz3QkJCAlarlYiICNRqNXNzc0xMTPj1QalQKDCbzTz11FNUVVXJ\nW9HOzs5y48YNPvzwQ7l1tMVi4bXXXmPXrl1yo53m5mYWFxexWq1ER0dTXFyM0WikoqKCy5cv09ra\nisPhwOfzYbPZflRCvdaka89gMJCZmUlaWhqNjY1+XZIpxSW16L548SJtbW1MT08/sDh6dnaWxcVF\nioqKAj4JWFxcZHJyErfbLY8YWa3WNZli/alSU1PZvXs3QUFB3Lx5kytXrtDQ0EBrayvj4+P4fD7+\n93//l7m5Ofbu3YvJZFrVMVChUMjdHcfHx9etrfojmQQEBQURFhYmb/IhtSqFrwuu/FVNarPZmJyc\nZHFx8Xtv0tKe62lpaezYsYMDBw6QkpJCX18fX331FWfOnKGrq2vNb/azs7Nr1l5W6vzm8Xhobm6m\nv79/TT7350hLS+P5558nOzub0NBQeej91q1bfP7554yOjvrtRvJNc3NzzM3NPbBS2Gg0YjKZCA8P\nx2g04vP5GBoaor293a8jAbGxsZSWlnL48GFKS0sJCwujra2NpqYmTp8+zc2bN9Hr9fI0waFDh4iL\ni5O3TK2trWVqakoekk5KSiIrK4uUlBSSkpJoa2vD4XDgdrtpamri97///bof48LCAmNjY3LyJRU9\nFhYWcufOHb8nARKpW+P3JeBSd8mNUHfhdDpXbdSk0WgwmUxMTU357drVaDS43W5qa2u5evUqFy9e\n5OrVq/L9WaVS0dLSwpYtW+S4BwcHGR4exmaz4fP5mJmZobW1lYGBgXU7jkcqCVCpVKjVauLi4igs\nLGTXrl3k5uaSmpoqb6PpcrmYm5ujvb3dL3sJjI+P09/f/70XmrQdbF5eHi+99BK7d+8mKSmJ1tZW\n3nnnHX7zm9+sU8RrQ0oC/NW29l7FxcX89V//tZwMOp1OJiYm+PTTT3nzzTcDugjq20gJrfRG2tHR\nwbVr1/w2d67VaiktLeXXv/613Lmwq6uLY8eOcfr0aXkr48zMTI4cOcKLL75IWFgYS0tLNDY28t//\n/d989dVXTExMEBoaSnFxMRUVFVRWVpKTk8MTTzzB008/Ddx9EH/wwQd+SQLGxsZoampi27Ztq6rT\nt2zZQnt7O/X19esek0Tan+GHPNClZZsZGRkUFxfLu04GMqnQTjo+qVDZn1sKDw0NUVNTw5dffvmt\nrX6lFsfW9kLSAAAL20lEQVSRkZGEhYWhVCq5dOkSJ06ckF/m7Hb7undTfWSSAJPJREpKirz7UlZW\nFlFRUQQHBxMUFCSPBLhcLnkTjPWsmpbmfuLi4khPT//eStbS0lL27t3Lk08+KffxvnXrFv/xH//B\n+fPn1ynqR5NKpVrVYMnhcFBbW8udO3dYXl72ezX9j7G8vMz169fZu3evv0MBvu6K+eyzz8pzy5cv\nX6ampoazZ88SHBzMoUOHKCkpISsrS64BuHjxIhcvXuTy5cs0NzczOzuL2+3GZrNx48YNenp6+PLL\nL+WpBamy+vjx43zyySd+Odb5+XmGh4cDps4Fvt5WV2oY1dbWtqoQ8NuYTCZ27tzJc889R1VV1X3L\nggPRwMAA9fX1vPrqq5jNZmJjYzl69Chvvvmm36bypB0kZ2Zm7utUq1arMZlMbN26VV7hIyW9tbW1\n8tSWVEewnqMxGzoJkPb2TklJIS8vjy1btvD444+TnJxMZGSkfKN3u93yxkMtLS00Njb6bUfBkJAQ\nMjIy2LdvH62trUxOTuJ0OrFarcTHxxMXF4fRaJQ3BsrMzGRlZYWOjg7OnTvH5cuX7+u6thFIVeIG\ng8FvMahUKsrLyykpKZHflODug1TaXGcjDIXeS6lUEhoaGjA3brVaza5du9i2bZu8/0JQUBCJiYlU\nVVWRkJBAZmYm2dnZREREsLCwwK1bt/j4449paGigp6dn1V4CUkHgzMwMo6Oj2O122tvbiY6OBqC+\nvp62tja/HGt/fz8XL16ksrKSkJAQ+S00JiaGpKQkzGYzCwsL6zYtY7FYSE5OJjc3l6KiIgYHBxka\nGsJms913XisUCrRaLVarlby8PA4dOsT27dtJSEiQa088Hg9utzsgr4nFxUVGRkbo6+sjKipKPi/8\nuT35/Pz8AzuLWiwWysrK2LlzJ0lJSczOznLq1CkuXbrk991UN1wSIA1daTQaLBYLaWlp7Nmzh8rK\nSrZs2SK/9UtFMdJuTcPDw5w+fZqzZ8/6pYOaz+fD5/OhUqlITk7m9ddf5+zZs7S0tGC328nPz6es\nrIzS0lK5tafU+6CtrY2vvvqKmpqaNenl7w/SUJi/mnpIw4WHDx+mqqpK/rnT6WR2dpb29nYmJib8\nEtvPodPpKCgoCJge6mq1mscee4y0tDTg7vWanZ1NcnKy3PwI7nakm5mZ4fbt29TU1HD8+PHv7Xex\ntLTE9evX/bYd8jf19fVRV1fHn/zJnxAVFSUnAUlJSWRmZmKxWFhZWVm3JCAnJ4dnnnmG5557jtjY\nWOrr67lw4QKTk5P3TXFJO2ZK06b79u2Tq9WlJagTExPMzMwE1JLMezmdTgYGBsjMzGR4eJj3338/\nYAp676VSqUhJSeGFF15g69athISE0NzczG9+8xu/r0KCDZgEaLVawsLCSE1Npaqqiueff57o6GhM\nJpPcZQzuvkFMT09z8eJFzp49y8WLF+U9tf2d2ZpMJsrKysjIyJCnJiIiIggNDZX3mLbZbIyMjNDS\n0sJvf/tbrly5gt1uD6gNkX6Mh9nd8IcIDQ0lOztbniaSkrLBwUEuXbrEyMjIhqsFgLs3c4vFIte8\n+JvP55M3BtJqtfIyP4PBgEKhYHR0lJ6eHlpbW7l06RKtra309/czNze3oaZhJFJjpsXFRXmUy1/H\nUVVVxeHDh4mPj0etVpOWlsavf/1r7Hb7fVMWer2enJwckpKSsFqthIaGolQqWVlZYXx8nAsXLnD6\n9Gnq6+ux2+1+OZ4fSrqWXS5XQJ5DYWFhZGdns3v3bsLCwuTC7qGhIb/t6nmvDZEESOvsq6urycvL\nw2q1EhMTQ0ZGBtnZ2avW4Eu7rDU1NXHnzh1aW1vp6OhgYGDAb9XSMzMzdHV1MTU1JQ/dSrUK0np6\n6Q1paWmJ3t5euef75OQkjY2NG/It9V7SXtmRkZF++f16vR6LxYLZbJaHzldWVrh27Rrvv/8+IyMj\nAbGu/sdyuVwMDw8zNze3qpW0vzidTj744AN6e3vJzs7GarXidruZmJigqamJsbExxsfHGR0dZWBg\ngJmZGXk+dCNyOBxcunSJ2NhYLBaLX2ORVolIfRaio6Opqqr61pqAe5NHqap9bGyM1tZWTp48yZ07\nd+jq6mJ+fj7g/22USqXcia+pqSngRkv379/PoUOHiIiIYHJykrq6Oj7++GOmp6f9/kIKAZwEqNVq\neZMUqbHCK6+8QllZmbzVpVqtlk9QaYOglpYW6uvrqa+vp6mpiYWFBb83FpmenubOnTvU19fLvaLN\nZjMqlQqVSoXX62V+fp6JiQkGBgZobGzk+PHjXLp0ya9xryWlUklUVBRhYWF++f1qtRqDwYBWq5Wn\ni1wuF52dnZw7d84vMa0Fp9NJZ2cno6OjuN1ueXOeoKAgv7xdO51OTp48SVtbG4WFhSQmJuJyuejv\n7+fChQvYbLZVVd0bncPhoKGhgZKSEgoKCu7bDXE9jYyM0NPTQ2pqqtwT5dtGiDweD16vV97ZcXJy\nktHRUTo6Orh06RIffPCB3I0vkEnbZxsMBjweD4mJifT09ARcElBQUEBRURFqtZquri7q6+sD6t4e\nsElAcHAwBQUFvPDCC+zYsYO4uDhMJtN9G2DA3ZNa6kL27rvv0tPTg8PhwOPxBEQW63A4aGtr41/+\n5V948sknOXDgAE888cSqdrpNTU2cOnWK06dP09vbu65LRISNy+Vy0dfXx8jICA6HA6PRSHh4ODEx\nMUxMTPjlRr68vCyPwEnnuNTuOBCux7XkdDrp6elhZGSExcXFb92Rb7189tlnOBwOXnjhBfLz8x84\nMuFyuXA4HLhcLoaGhrh9+zYnT56kqamJ4eHhdV859VPp9Xr5OMfGxu4rKg0Ud+7coaWlhR07dnD7\n9m16enr8HdIqAZsEhIWFsWPHDkpLS8nMzMRgMKBSqeRlQ319fdhsNpaWljh//rw8t9jd3b2qiUSg\ncDqdjIyMcObMGVpbW3nvvfdW3SAnJiYYHh6WbyaBFv9P1dPTQ09Pj7yky18WFhbo7u5mZGSEtLS0\ngNp17Ofyer20trZy7tw59uzZQ2lpKTMzMywtLTEwMOCXG6M0zfWok6Y63n33XS5cuCCvSBoeHl73\nplMDAwN8/vnndHZ2Ul1dzc6dO+WVMHa7Xa7HaGlpoa2tDZvNxsLCAjMzMwwODjI3NxcwTbJ+CLfb\nzfj4OFarFZvNht1uD8gpvbq6Ovr6+rBarXR0dDA4OOjvkFYJ2CRAauBy7dq1VV+aVLn6zSRgdHQ0\nYKtY4euCKemhuFk0NjYSEhLC1NSUX4/d4XDQ399PTU0NQ0NDhISEsLKyQlNTk1/iWWvt7e2cO3eO\n3NxcEhIS2LNnD5OTk3LS+agklYHG5/PhcDgCYtWCtKlXf38/8/PzDA4O0tnZiUqlYmFhQU4C2tra\n6Orqwm63b+hEbWFhgZqaGlpaWpiYmAi4zaUkvb299Pb2+juMB1IEwrCPQqHwfxDfw+fzPXCiL9Dj\nF7H7z4PiX+vYdTodeXl5/MM//AMlJSUYDAZ6e3v5t3/7N37/+9//pJUPG/m7F7H7z0aO/1GN/bv4\nfwNmQRB+NpfLRXd3N//8z//Me++9R29vL0lJSSQlJREZGen3bVYFQQhMATsdIAjCD+f1euUWu1L/\ng8TERG7fvs3S0tKGKPQSBGH9iemAH+hRHSYSsT9c6zUd8DBs5O9exO4/Gzn+RzX27xIQSYAgCIIg\nCOtP1AQIgiAIwiYlkgBBEARB2KREEiAIgiAIm5RIAgRBEARhkxJJgCAIgiBsUiIJEARBEIRNSiQB\ngiAIgrBJiSRAEARBEDYpkQQIgiAIwiYlkgBBEARB2KREEiAIgiAIm5RIAgRBEARhkxJJgCAIgiBs\nUiIJEARBEIRNSiQBgiAIgrBJiSRAEARBEDYpkQQIgiAIwiYlkgBBEARB2KREEiAIgiAIm5RIAgRB\nEARhkxJJgCAIgiBsUiIJEARBEIRNSiQBgiAIgrBJiSRAEARBEDYpkQQIgiAIwiYlkgBBEARB2KRE\nEiAIgiAIm5RIAgRBEARhkxJJgCAIgiBsUv8fwz/y3MbKPeUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc1182de8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label: [5 0 4 1 9 2 1 3 1 4]\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "for i in range(10):\n",
    "    plt.subplot(1,10,i+1)\n",
    "    plt.imshow(train_img[i], cmap='Greys_r')\n",
    "    plt.axis('off')\n",
    "plt.show()\n",
    "print('label: %s' % (train_lbl[0:10],))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we create data iterators for MXNet. The data iterator, which is similar the iterator, returns a batch of data in each `next()` call. A batch contains several images with its according labels. These images are stored in a 4-D matrix with shape `(batch_size, num_channels, width, height)`. For the MNIST dataset, there is only one color channel, and both width and height are 28. In addition, we often shuffle the images used for training, which accelerates the training progress."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "\n",
    "def to4d(img):\n",
    "    return img.reshape(img.shape[0], 1, 28, 28).astype(np.float32)/255\n",
    "\n",
    "batch_size = 100\n",
    "train_iter = mx.io.NDArrayIter(to4d(train_img), train_lbl, batch_size, shuffle=True)\n",
    "val_iter = mx.io.NDArrayIter(to4d(val_img), val_lbl, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multilayer Perceptron\n",
    "\n",
    "A multilayer perceptron contains several fully-connected layers. A fully-connected layer, with an *n x m* input matrix *X* outputs a matrix *Y* with size *n x k*, where *k* is often called as the hidden size. This layer has two parameters, the *m x k* weight matrix *W* and the *m x 1* bias vector *b*. It compute the outputs with\n",
    "\n",
    "$$Y = W X + b.$$\n",
    "\n",
    "The output of a fully-connected layer is often feed into an activation layer, which performs element-wise operations. Two common options are the sigmoid function, or the rectifier (or \"relu\") function, which outputs the max of 0 and the input.\n",
    "\n",
    "The last fully-connected layer often has the hidden size equals to the number of classes in the dataset. Then we stack a softmax layer, which map the input into a probability score. Again assume the input *X* has size *n x m*:\n",
    "\n",
    "$$ \\left[\\frac{\\exp(x_{i1})}{\\sum_{j=1}^m \\exp(x_{ij})},\\ldots, \\frac{\\exp(x_{im})}{\\sum_{j=1}^m \\exp(x_{ij})}\\right] $$\n",
    "\n",
    "Defining the multilayer perceptron in MXNet is straightforward, which has shown as following."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'mx.viz.plot_network(symbol=mlp, shape=shape)'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a place holder variable for the input data\n",
    "data = mx.sym.Variable('data')\n",
    "# Flatten the data from 4-D shape (batch_size, num_channel, width, height) \n",
    "# into 2-D (batch_size, num_channel*width*height)\n",
    "data = mx.sym.Flatten(data=data)\n",
    "\n",
    "# The first fully-connected layer\n",
    "fc1  = mx.sym.FullyConnected(data=data, name='fc1', num_hidden=128)\n",
    "# Apply relu to the output of the first fully-connnected layer\n",
    "act1 = mx.sym.Activation(data=fc1, name='relu1', act_type=\"relu\")\n",
    "\n",
    "# The second fully-connected layer and the according activation function\n",
    "fc2  = mx.sym.FullyConnected(data=act1, name='fc2', num_hidden = 64)\n",
    "act2 = mx.sym.Activation(data=fc2, name='relu2', act_type=\"relu\")\n",
    "\n",
    "# The thrid fully-connected layer, note that the hidden size should be 10, which is the number of unique digits\n",
    "fc3  = mx.sym.FullyConnected(data=act2, name='fc3', num_hidden=10)\n",
    "# The softmax and loss layer\n",
    "mlp  = mx.sym.SoftmaxOutput(data=fc3, name='softmax')\n",
    "\n",
    "# We visualize the network structure with output size (the batch_size is ignored.)\n",
    "shape = {\"data\" : (batch_size, 1, 28, 28)}\n",
    "\"mx.viz.plot_network(symbol=mlp, shape=shape)\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now both the network definition and data iterators are ready. We can start training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Start training with [cpu(0)]\n",
      "INFO:root:Epoch[0] Batch [200]\tSpeed: 26634.35 samples/sec\tTrain-accuracy=0.111150\n",
      "INFO:root:Epoch[0] Batch [400]\tSpeed: 26589.34 samples/sec\tTrain-accuracy=0.112450\n",
      "INFO:root:Epoch[0] Batch [600]\tSpeed: 31287.89 samples/sec\tTrain-accuracy=0.140250\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=2.152\n",
      "INFO:root:Epoch[0] Validation-accuracy=0.290900\n",
      "INFO:root:Epoch[1] Batch [200]\tSpeed: 29611.20 samples/sec\tTrain-accuracy=0.429100\n",
      "INFO:root:Epoch[1] Batch [400]\tSpeed: 30116.49 samples/sec\tTrain-accuracy=0.749500\n",
      "INFO:root:Epoch[1] Batch [600]\tSpeed: 30836.50 samples/sec\tTrain-accuracy=0.827900\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=1.995\n",
      "INFO:root:Epoch[1] Validation-accuracy=0.845600\n",
      "INFO:root:Epoch[2] Batch [200]\tSpeed: 27080.45 samples/sec\tTrain-accuracy=0.861800\n",
      "INFO:root:Epoch[2] Batch [400]\tSpeed: 26364.98 samples/sec\tTrain-accuracy=0.885150\n",
      "INFO:root:Epoch[2] Batch [600]\tSpeed: 26314.78 samples/sec\tTrain-accuracy=0.903400\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=2.265\n",
      "INFO:root:Epoch[2] Validation-accuracy=0.913400\n",
      "INFO:root:Epoch[3] Batch [200]\tSpeed: 26375.76 samples/sec\tTrain-accuracy=0.919050\n",
      "INFO:root:Epoch[3] Batch [400]\tSpeed: 26289.85 samples/sec\tTrain-accuracy=0.925250\n",
      "INFO:root:Epoch[3] Batch [600]\tSpeed: 26381.15 samples/sec\tTrain-accuracy=0.933000\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=2.284\n",
      "INFO:root:Epoch[3] Validation-accuracy=0.941100\n",
      "INFO:root:Epoch[4] Batch [200]\tSpeed: 26409.37 samples/sec\tTrain-accuracy=0.941800\n",
      "INFO:root:Epoch[4] Batch [400]\tSpeed: 26324.14 samples/sec\tTrain-accuracy=0.944050\n",
      "INFO:root:Epoch[4] Batch [600]\tSpeed: 26346.50 samples/sec\tTrain-accuracy=0.948650\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=2.283\n",
      "INFO:root:Epoch[4] Validation-accuracy=0.952600\n",
      "INFO:root:Epoch[5] Batch [200]\tSpeed: 29269.77 samples/sec\tTrain-accuracy=0.955950\n",
      "INFO:root:Epoch[5] Batch [400]\tSpeed: 26427.26 samples/sec\tTrain-accuracy=0.954100\n",
      "INFO:root:Epoch[5] Batch [600]\tSpeed: 27040.70 samples/sec\tTrain-accuracy=0.959350\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=2.187\n",
      "INFO:root:Epoch[5] Validation-accuracy=0.958300\n",
      "INFO:root:Epoch[6] Batch [200]\tSpeed: 26420.80 samples/sec\tTrain-accuracy=0.963850\n",
      "INFO:root:Epoch[6] Batch [400]\tSpeed: 26330.90 samples/sec\tTrain-accuracy=0.961250\n",
      "INFO:root:Epoch[6] Batch [600]\tSpeed: 26363.10 samples/sec\tTrain-accuracy=0.965800\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=2.283\n",
      "INFO:root:Epoch[6] Validation-accuracy=0.961900\n",
      "INFO:root:Epoch[7] Batch [200]\tSpeed: 26299.18 samples/sec\tTrain-accuracy=0.969750\n",
      "INFO:root:Epoch[7] Batch [400]\tSpeed: 26307.93 samples/sec\tTrain-accuracy=0.966400\n",
      "INFO:root:Epoch[7] Batch [600]\tSpeed: 26250.44 samples/sec\tTrain-accuracy=0.969750\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=2.290\n",
      "INFO:root:Epoch[7] Validation-accuracy=0.964000\n",
      "INFO:root:Epoch[8] Batch [200]\tSpeed: 26373.71 samples/sec\tTrain-accuracy=0.974650\n",
      "INFO:root:Epoch[8] Batch [400]\tSpeed: 26281.62 samples/sec\tTrain-accuracy=0.971250\n",
      "INFO:root:Epoch[8] Batch [600]\tSpeed: 32782.91 samples/sec\tTrain-accuracy=0.973750\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=2.138\n",
      "INFO:root:Epoch[8] Validation-accuracy=0.967000\n",
      "INFO:root:Epoch[9] Batch [200]\tSpeed: 26263.50 samples/sec\tTrain-accuracy=0.978650\n",
      "INFO:root:Epoch[9] Batch [400]\tSpeed: 26157.16 samples/sec\tTrain-accuracy=0.975000\n",
      "INFO:root:Epoch[9] Batch [600]\tSpeed: 26077.45 samples/sec\tTrain-accuracy=0.977950\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=2.303\n",
      "INFO:root:Epoch[9] Validation-accuracy=0.968300\n"
     ]
    }
   ],
   "source": [
    "# Output may vary\n",
    "import logging\n",
    "logging.getLogger().setLevel(logging.DEBUG)\n",
    "\n",
    "model = mx.model.FeedForward(\n",
    "    symbol = mlp,       # network structure\n",
    "    num_epoch = 10,     # number of data passes for training \n",
    "    learning_rate = 0.1 # learning rate of SGD \n",
    ")\n",
    "model.fit(\n",
    "    X=train_iter,       # training data\n",
    "    eval_data=val_iter, # validation data\n",
    "    batch_end_callback = mx.callback.Speedometer(batch_size, 200) # output progress for each 200 data batches\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "After training is done, we can predict a single image. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFfCAYAAACfj30KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnX1bGknTRxtFAV+y2fv7f8ZnN1ER8e35Y7fY3xRVPQ0R\nG8k51zXXDGMCxuihrK6umry/vxcAAOjDWe9PAADgdwYJAwB0BAkDAHQECQMAdAQJAwB0BAkDAHQE\nCQMAdAQJAwB0BAkDAHQECQMAdGTa+xMopZTJZMLeaQA4Od7f3ydjf4ZIGACgI0gYAKAjSBgAoCNI\nGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgA\noCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAj\nSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gY\nAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACg\nI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNI\nGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI0gYAKAjSBgAoCNIGACgI9PenwD8x2Qy\nqT5u/TOH4v39fe97uzwnwO8EEu7IZDJpPs7Ozpr+jMdLLpNeiwzf3983x9vb2+CxP/xzjl3X7iFq\nOGWQcEdMnP7I7rccFhm3Sm0XKb+9vYWHSdnf86/Zep19DOAUQcKd0Oj1/Py8nJ+fN137w3/MouFI\nYNk9Yyxqfn193Rxvb2/V67EIuXaoxGufD8ApgIQ7YtGrCXQ6naZnfx09nk6n5ezsrCm61Ht2v3b9\n/v5eXl9fy8vLS9O5lrLw9zWafn9/L5PJZHDPf04ApwQS7ohGwibRi4uL9Noe69lfn5+fj0aaLbnb\n6Pzy8lJeXl7K8/Pz6LUXa3bWVMZkMhmI1+7Z1woRwymChDuh6Yizs7OBSPW4vLzcuq6dp9Pp6OKZ\n3i+lLl47v7+/l+fn561jvV5vrqfT6ea6ljP2j3Vx0QSsvL6+fsZ/CUAXkHBHfE7YRHx5ebl1zGaz\nrevo3sXFxVaUOXZdSi5evV6v11vHxcXFRsbr9XqTl7b8cOvx+vq6VW5nqQk7AE4RJNwRzQl7Ac9m\ns/CYz+fp4/l8PpCwjzqzdEApbRUL6/W6PD09DQ67Z/I1WapcvWz9tQlYI9739/dNfhsBwymDhDvh\nqyM09+sFO5/Py3w+L4vFYnPtD/vY5eVl+qt/lBbYZRHv6emprFarslqtNlG3ClglrILV89nZ2UC8\nlo4wAfvXJBKGUwcJd8SnIzT36+Xbesxms+YUwK4lZCbfy8vL8vj4OKjI8DXKWTmbpSrs3+6jX11Y\n1Hyxfb1YnINTAwl3wuSS5YNNwFdXV2WxWAzOtetIwrX8bCnjdbul/JNe0EVCLZNTCWskrCVrteP8\n/Ly8vLyUs7OzwTmKgpHw4Ym+xtnXnf+PXwcJd8RXRlgEbFHt9fX1QLRR1GvSVjF60VoKwKcjMglH\n9/RjUSrFR/FetBoN+3taY5xd25kf+v1pketYNU3Lb1D6vHoPYpBwJyKJaRrC5Htzc1Ourq7SvLBW\nRVidcCbdaLGulPHtwv4HKctl6+fvRevP/nqXTSCwO16OtXu1RdToN6vsDVvLDfl/y0HCHbF0hEpM\nZXt9fb05fBWEVkhYNGxCzKogIhGX0t7XwT7nWh57NpuV9Xq9Jd7ataYsomu9xw/zfoxFqXquff39\nvdoicCll8D0GMUi4I15mPh1hkfD19fWgNji61lxt9qtjJORS6j0l9DwWAVut8PPzczUPHZWt1SSs\nj/lh3p+xlJPd092P2dmuo/9Lv+mGzTZ1kHAnot1yPh1hUfDNzU24gUMX8jQnXPv1MPvBazlnAtYd\ndPbDuUuFxtgiHpHwrzOWx9VDd0LqG6s/W2mhppeiem/KC+sg4Y5E6QiTsEbCNzc34Tbm6Ih6R5RS\nX2hrzRf6HLZFRNHRUqts12PVE3og4f1oeUO2Qzfl6LWvCy+lbKpZrKLFv6ZWuPB/F4OEOzG2MGc5\nYZXwWJMfKxnLFtV2kW90fX5+Xk0V+KhoLB9tH6tJ10uaH+T9yNJR0bVJ12/MWa1WW+1StZywlDKo\nFbcdj5qagG2QcEeySFjrfk3Evn1l1urSSzg7Z7LN7mnUmi2y6Xks6vIS9rLNStqQ8H5kb4LR2eT7\n+Pi42ZiT9auOdjPa81iVDtRBwp3IcsK+OsIi4ZaG7navNbJVMrnp/azMLbvXeoyVr+k17MdYakgf\nPz4+luVyOVjs9QLWEke/mebt7a2cn59vcsTkhOsg4Y601AmbiHVn2thRyv47nGofr+WY/b2xx3qv\npZbYzrAfLQukdr1cLgeLvbolvZTh/1n0f2vP43dRQgwS7kS0bTnasmwVEvoNrXPookGgx0JLWqSU\nUk1tIOGPoVYi6A+rOdffrnxfEKuC8RK2KNhK1SwoYGEuBwl3QqMJ+4a2xZDlclmWy2W5v78v8/l8\nqwG8F7G/Z8+vr+Wv7azSjq5r98bO/rVb8JOjbYHHf+1q+I+PvTH1eOP6VSFZPraFaDJ3lpIwCdtv\nZc/Pz4NrrYAx0eqbv08xWfUE5CDhjry9vW0EbKVAtiDy8PCw2RlXSgnTDtFUZv/DoNfRY/9DVHvc\n+rGalLNrRZ9HI/wommpZVPSvE73uoUW8r3RraaXWf4f/eo7lhLX+2wTsSxAtmtbX1Od7eXnZiqIh\nBgl3QisDfCRsErbdcBYNRnnh6F5WkRAdmUS9/FpSIdnfqUnevhb6Mbv2+W0v4dZFx5Y3gUOKYlcB\nj73RZFUJtX+bCtj3FvFnlXBNwFr54CPg6XS6iYKRcB0k3BH9lc12Jlk6wo8r0mqI2jn6Act+2Ezu\nY1LNou6xiDwSc/RY8dGzTtfw0Xx2zuTUkm6JBK73fvXxGLu8ybT8e/znoSLOvkc0CtYUhK8B92/0\nPr0WtTiFbZBwJ/Sb1rb8aiSsq9P2Q9NSojaW7/P3W6WaRd/ZdfRcKoKxKg6TRJZK0b9b+1gtPXJs\ncqjl7qN7LWkfTRXY4b+2XqYvLy/l8vIyjICjRb1IwFrWdkxf42MECXfER8ImYV8e9Pr6GtYJ+2M6\nnQ4kXFsNt/KisdSGl2wk/+wNInouxWSgZFFsJtpa3tv+fiTdSM49qck2O7e8ufi0T8sRSTirqDAB\na1Mf3/D/GL6+xwwS7oSPHlTC2hd4MpmUl5eXQclQ7dqkHdXYRudaesPf09fyrx099s9n/24jEq6m\nE/zHa8LdRcKZjHuQ5X9bo3w7j13rc4x9He170kScRb/+e9i+j+37V7+HEXEOEu5Elo7wAi6lbEUX\n2RZmL+GWngwtEbYXrX9Ne2wLMvrn39/ftyJg+6G0H+RdF8taorlsIbC2aPjZRKmY1ojfqL2xRBKu\nvYadX19fy+XlZZp+8BK2CPj5+XnQy8RX7EAMEu6IRh0WQaxWq0H+9P39PZVwdJiEaw12NLppiWqj\n1/WNg6bT6eY5LY8dRailDFfrW3611rN93Wo5TRVx6/HZRAK2+y3RqtHy76qJOLqXvWn7igj7syZg\n+x7ONnpADBLuRLSYsVqtBjlQ+7hFyF56/t7FxUU5OzurzmrzkxFqEXYm34uLi03UY5+DVXHYoX2N\nDRODrs5H6YextIGXrr5h6cdKKdWyuZ6/KrdEwWMpl5p0x95kaq/v1xKyN9NS/pOwpdN0PQMJt4GE\nO6LpiNr20PV6PZCuP6scVcJjh0q45fA9jS1K8gLOfmBVfvZnLWWRCSTarJFJN4oWo+cYK5M7NPtE\nwdExVpe9b6Rv35dZOZry8vJSnp6eBiWVUSSMiHOQcEdMIhoRn5+fb2osVTxW/D6dTgfpCR8Vm4Q1\n8v2VSFgfq/x9FBxdj6UvfIOYqKTNn0sZ7whm16WUreeqvcah/69rj+1eLcXi741Vquj1r3ze/nPQ\n79mo10ctcoZtkHAn9Jtac7V+McP+TGvKIMoJ1+a2jVVbRDnhKA2yS0Qd5bFbDy+CWhleKSUtl/Ml\neIeUcJR/zT4WvZlk57HfjKbTaRjpt/xbI/lmfZ5r1RNRDhqGIOGORJGwF4L9mZos9XFrdYRfmGut\njMii5JbH2cd8LXK2CcRSF60leCaglvK7Q6Ykaoth/uNjG230437moE7e1pz4+fl5uDg3JuNa9Os3\ncHgZI+B2kHBHfJQRlRS1lJJ5sfiIJYpi7Iclk1L03LtUUmSRdHTO/i3RfZXw2DGZTMLnio7PTEd4\nOUUSzqJ7vWctT+3Q+X6llE1td5Sb19f1//ZaCsLLOBNwlkOGbZBwJzIB6w+IfjyLFL24rPrA/wBH\nj21xJ9sxl0WjreLMpD0m8tpjH41FZ7s2Cbe8CRx6ca4m3uj/O3vj1P9HHYOl4+ftN4Dp9J+yQXt+\n/d7aNSWRybe2mQMRt4GEO+K/uXXVX38gNVfcktus/QrrH+siVXZkv9bXRD0m5lqao3b9/v4eVnlE\n1R+TySTMQfu0iOWlD/n/nJ39dS3v6mV8fX29GT9vb6ql/JeCuLi4KLPZbPP/rJHvmJSz/HT0Jpjl\nhhFwG0i4I/qD5+9Zvlaj4LEVfpXw2KKOnVsqEvyfGZN1LZ0RPc7K4aL7b2/DPgW1a5NwbeHKrg8t\n4RYBR6mWSHSWcliv15uPZwLWj5UyFHAtIs5SErU3hmxRDuog4U7oN3d0T+XrZThW++p/FfQ/TPo4\neq6W1xgT9liErOdslT86v729bXZo6XbZ6N5kMknL6fy98/P9y7ha/69rAi5l2NCp5TwWAZusowg4\nO9vn5L8fo/SIf4PIRIyM6yDhjqiE9Rs+ku5YIb7f0JBFWv5e7Xmyey1HlmeO7nkx1sSpEradWl7E\ndm8ymWw2EPiNJv7xoSTc8n8QSbi2wUblpwK2r9VsNiuLxWIQJbdGwPp5Z5HwriKGOki4Iz4itUWV\nSH72WM/RPc39+XN2b9fn9+fo3ljqQo/pdDqQY+3afg030erZX5vgrWyrdv0ZkbBeR0LWVEuUZvGH\nX4RTAVuqQt/ks1xwTcq7VkZkv4FBDhLuSCREY5+SKf07H/mNv8/nkuWKo8cqWT38PYuETbJ6PD09\nbd07Ozvbes6opvby8vLT0hE1IZuENa2SnW0dwRYZVcCr1SpcsCtl+CZd+39tyQnXomBNg9jzQQwS\nPlL2+aY91Df6vp+Lpldqi4C1vxv98GeRr6YmbGEuEo3PvVsd9qEYk28tJ2zRrK4dnJ39s3NQK2Jq\nqatSYuH6/1eLiFWymmPXN7zVarU59A3QvvYqZgRcBwnDwYhykXZf5ayLkJE8vZCjxTj/g2+/qttz\na+2wPq8dn5kTrj32EWUpw6Gnmn7QSF7L7zTnHn0t/f+FPi6lDCSsAn56ehqM4Foul+Xx8XEjY/8m\nGEXEsA0ShoNSy0ubKH2ttOYsvYBNwtGv6ZGE7doL2D//Z5ao1a6jCNlXn9gbhl9krHUwi9YK9PNT\nfG46ioAfHx83h0XDFhFbGkT/LyAHCcOnoD/oXno1Aft85Pv79maNSMA+5VCLri1aPvS/P1owra0L\nKD5Sn0wmg/aRkYDt0L9Ty9Pa4ywSzkSsKQn/mwlVEuMgYTgIYz90WgliaYOoTtXnbU3CY9uWNR0R\npSB8rrnHZo3onNWC+2tddPQbW2rpCP2css/N54MzAVs6QqNgnxcmHTEOEoZPIftBtJzkZPLPQFMf\nAZt8VcK1Lb0+HWEi1tdTCZ+fn2+2hX/Gvz2Tr6F11KWUTT8QX2OtOeEoEvbpiOy1o/SHLoDWcsJ2\nqKSzSBhykDAcFM1rehmZIPyGlUjAJiEfxUabCPRX4Cy/bM+psvqsr4Vee0FZ1zPtBayLcdr3Yjab\nDWqpfSSsfz9KPfjDvm5RJGyRbpSOiDbN+OoIRJyDhOFgRCvv/r6dvQy0WkLzm36hLlq4s3v6eirl\naEv4Z0jYPpfo2qhVRNiW5Kh+2kvYv7lEb4JevirhKCfs5btcLstqtRr8uWihFAHXQcLwKYwJ2a59\nrtiv8JeyPegze2xRtop97LkP+e8fu+d3HFr6xUvYNpv4uW61EjX/hqdfb3/OqiMiEa9Wq62t1pqf\nJx0xDhKGg1LLBZdSBmIY2wodVU7o4+ha/65GmdHzfibR10XfHKxjnKHd5qKdhWMi9q+ZRcK+L8RY\nnfDT01O6lZkStTaQMHwaLdFgKfttk86e6yOe97OwaNfy4Pom4tMRftt1SyTs8SL2Ao6qI/zCnEm4\ndpATroOE4eg41A/ssYkgelNQ4WpPiNlsVubz+WCaxtXVVVksFpvxRiZmvzhn6IJn7dBdcFFPjqwC\nImrgA+MgYYBPQoUYXUeLb16+19fX5fr6ulxdXZWbm5uBiK1aQsvVfL7dR7nRxpeHh4etLclZ+ZnP\nKUepIqiDhAE+gVqO28467km7vXkJ39zcbGR8fX09kLC25tQdc17CWS/m5+fnslwuBxL2GzF8+Vkk\nYETcDhIG+CSizmb6WBfffBrC0g8m3tvb20Fqwkt4LBKO2oHaoZGw7ojTMrRsi7hPQyDgcZAwwIFR\n2dZEbJHwWDri5uam3N7ebu5HOWEbXqoLc1EkbIK1Bbenp6fy8PAQ9oWwRvGZhLNIGOogYYBPJpOy\nLshZOsIEG0nYImUTdWs6wpedWbWD5X99OmKsTSUS/jWQMMAnUBOv9obQdEQtEr65uQknhLSmI1TC\n2gfi8fFxa2FurE1lVqcdPYZtkDDAJzEmYJ+OiErTNBKuDUj1JWo1Cevmi4eHh1DCWWma9v2wMwty\nu4GEAQ6ML0eLBFyLhBeLRShh3aCRXY9JWLchm4AzCWdtKiPpIuJ2kDDAJ5ClIvTYpUTt9vZ28He0\nI5zfNWfUImHLA5uAd6kTLmV8WgfkIGGAT6KWF1YRt1ZHRFF1dC4ljoSjXhD39/ejdcKRhGF/kDDA\ngYnyvj5qPT8/31RDaNWDLrzp9cXFxWjpm5L1iIh6Q2gOOBukyoLbx4GEAQ5IVH6WHZeXl2k/CF92\ntksLTi9glbDvJRG1oxzL/8KvgYQBDoxvyGNnf8xms42EF4tFOsjTyzcTcdQuNBKx7wecDU8lAj4M\nSBjgwGQbMXxtr25P1rSEb1XpJ2bo2eNLx1pEnPUEpkPaYUDCAAcmak+p0zG0Hlgb8szn84Gko7FF\n9vwebZI/Jt9aJOzn9umkDET8MSBhgAPid8P5qgeLeDUK1nSEtafUfhCtg0mjnWu+4brmfqNUBJHw\n4UHCAAdGm/NkW5K1BM3SET4S1nSEPncLtSkaWY9hcsKfAxIGODA6Ny6r//XTMnwkrOmIsaqISJJR\nBFxLSdSqIxDxx4KEAQ5MLSfsN2LowpxGwtnCXCnjudlsUS6aKRflhP0BH8vZ+B8BgH1RAfu5cdFO\nuFok7HPCLZURteqIrDY4q5AgEj4MSBjgwETpiFokvEt1RKuIa1FwFg2TE/4cSEcA/AK1UrHJZDLo\niuZL0/yinB/amS3KRROUoxFDeq19gXVrsvaEyLYpZ1M04GNAwgB7MDaqyI7pdDqQrpak6dQMO6Lt\nyibgqEdwtv3Yb03++fNn+fnzZ7m7u9s06cmatmfd0lTE8HEgYYA9aO1gps3ZvYy9gDUPHG1X9rlg\nP8K+dqiAWxu3R9EweeGPBwkD7InvB+xbUp6dnQ0W4fyhka9J2HdM06oIXyNskbBNT9bZcX6k/T4S\njgRMJPzxIGGAHYmGc0YN1q09ZWsk7BfionywpiOy/sBRW8q7u7s0HeElrHPk2DF3eJAwwB74huwa\nrep1JmAfBdtZu6pp1zU/wr6U/xbl/JSM6DD5RpGwzwlHfSP84hx8HEgYYA+iRu1Zj+AoAs7SEVmv\n4drCnEbCNqrIJmX4AZ4PDw+DSNiiYT85I9qkQSR8GJAwwB74SFglrFGsj4THFuZ8JO0ncWQLc35y\nskpX58bZdS0n7KNef0bCHwsSBtiRSMB+XH00tr6lRE1zv5pv1mvDR8Lr9XoQ+Vr64e7ubisytnOU\nE359fd2qgkDAhwMJA+xBbWyRLqrVomCfD14sFqMlb9nCXBQJ393dbRbk7L5Fvnr2OeFolFH0GD4G\nJAywJz5i9WOLIgFHi3MaCUcbP/SxXZdSzwlbJPzz58/y48ePTbSrka9eewkbjLI/PEgYYA98JOxz\nwjq6SDdg6Hbk6GhBUwTRxGQdYW85YU05RFuXtSwN0X4uSBhgD2qLcr5Bj9+CnNX9lrI9DUPxH/O7\n4lTGevgaYF+GRmOeviBhgB3RVEG2KOcjYF//WxNxKXEbSr1WCfudcl7GUV+I2nZk+FyQMMAejEXC\nPh8cNWePys5KGYo2O97e3gbjiDTK1W3LPuUQNedBwH1BwgB7kOWENRL2OeCoIU8tCh5rU+kjYS9i\njYRV1lm/YOgDEgbYkbGNGn5BLuuKFolY0w66ScJfZxL28rUj665GTrg/SBhgD7LKiCgl4ZuzR/Pi\nsnREtn1YS9Oi3LCXsR/cmeWE4fNBwgB7EEXDUSrCV0dk6YgsJeHFG82Gy/LBetQavpOO6AsSBtiD\nWjrCb9TQOuBoYc62ImtKwkfD0fQMX55WS0lkk5NJR/QHCQPsiO5cixbmWnPCWSTcMpgzE3CWkrBN\nGNFCHxLuCxIG2IOxSNhXR3gBZznhqB44G1EflahF+eCnp6dBH+Cs7A36gIQB9sBHwSriaLtyLRLW\nzmilbG9LjubIeeH6zRpaE/z8/DwQcO0Mnw8SBmjEpyFqkbDvHzFWmjaZTAbizRbc9HE0NTmajmEg\n3OMECQMk+IoFvR/1Es4a+OyyZTnqEay73jTNoI3ZrSVl1pg9EjAyPg6QMECAitHLWHv77hoJ17Ys\na/7XS9gObUNp8+KsOXsUCWfSRcDHAxIGcGQCjtIRUSSseWGfC47ywfoamgf2Azx9Q/axsfXZRgx6\nBB8XSBhAyKTrz7VewlkkfHFxUZ0ZZ7KM0hHaI9jOYznhaDIyAj4+kDBAQIuAs2nLvp3lWD7YD++M\n0hGW991lWCdbkr8GSBjAkYlXr8ciYYt+bZdclhP2gztrkbBOTLZ88MPDw5aEfavKCKR8PCBhgIAW\nAbdEwrpVWY9st5yWqGUDPG2Kskl5bGGOBbnjBgkD/IuvgvD54drCXCbgy8vLwQ45nxOOStT87Lho\nbtzd3d3WBOUoJ2yRMPI9XpAwgBCJeCwaHhOxCTeScFSiVktHaCTsJydHOWHywccPEgZwqGRVun6u\nnE8xaGWE5oV9SZrWGWdRcCRhjYbv7+8HNcS6fdk3bIfjBgkDlO2tyBa5Ro/n83m5ubkpV1dXZbFY\nlMViUebzeVqO5qPpUv4Trl3XRhZl05O9dBlZ9DVBwgD/4qNcvxHDrheLRbm+vi5XV1fl6uqqzOfz\ncLR9bXJGKWVTPjaZTLZK06IWldHcOC9ihnd+PZAwQBlGwr6kzK7tbBK+vr7eRMLWMS2aI6evYfhe\nDpPJJJ2WoQK2KNjmxrWMsYfjBgkD/Iv2gfCLa5rjvbq62qQjLCVhkbCJWEvRlFozHauIqKUiVMR+\nxpxGwtFuOThOkDBAKYNFN5OwpRe0L/BsNitXV1eDSFjTEVHP4FJKWK/rhRz1DK6lI3R4Z5QPJhL+\nGiBhgH/xkfDl5eVGriba+Xy+kbBGwtFUZUtH+J6+fmqGXbfIVyPhaHIyC3NfDyQMUIaTMvzYesv7\n2qES9pFwtDBXynARrjZDrja2yOeGdQCov2Zh7uuAhAH+RUWsEp7P5xv5airCl6hFEtaFuSgKVnnu\nko5Yr9db05Kj6clI+PhBwgClHglrCsIfVqLmpyprJGwyjCYq+wnKLZOTbYNGltZAwF8LJAzwL1lO\nWNMQNzc3aSQcTdEwCfu8cC0VEaUhooU5e66WMxwvSBiglMGuOKtwsOjWR8I3NzcbKUflaSZhe863\nt7etxu1+i3Jtk0YUDT8/P3f+isFHgYThpMma7/j7mvvV6DdalMs2aETblHeZoKwji6xXhEnX74aD\n0wEJw0mjjXKis11bFYQKuCZjXYjTnsG6VdnImvL49IKfG2e5X5M0Ej5NkDCcLFEDdt+YR5vyZNK1\n6DeKgv0gT98juNYZzU9Qtqbt2iPYR8J+gjJ8fZAwnDR+DFF2tEbAJmJt2m6RsO8TXEosYZWvyVb7\nBfu5cb49JRI+LZAwnCw+9eB7AOtj3XQRiVij4MVisdU7OOqcVko9EraI13oEq4S1Sbs2akfCpwcS\nhpMmErDvinZxcVGNfiMRaxmaF3urhHVunAm4Fgkj4dMECcPJkk1FjqZfaG+IsXTE1dVVmtaIcsI6\nQdkiW01B2KQMJPx7goThpPGz4KJBnJeXl6NRsD9qEzh2mRunwzt1gjLpiN8HJAwni4+Eo0GcuiFj\nl5RENIfOz6QrpS0nbBHw/f39JgJmYe73AQnDSRNNRI4ErJURmZA1J1zKcFKGbvxQWiNhnaCsgz0p\nUTt9kDCcLLVI2NIQvlew35CRVUe09mwYK1HTqoj7+/vB+CKdJUc64nRBwnAS+Ai0lBLmgb1sTa7W\nE8L3g9AtyRZVG1HHMn+vpReEbtzQyJdpGb8HSBi+LFEKQK81+tX0g+8NHPUI1m3JvkG7EbWS9Oex\neXHRoX/HixgBnx5IGL4kvimPv1dKGUxOtq5ofkSRRcDaI7gm4awbWnR4CY8N76xNUCYSPl2QMHw5\noq5o+tiubTFOc8CaijABm4T99GTtCxE15vECjkYNRfKN0hE+DaEVEX6EPRI+LZAwfEmylpR6+MU4\n3x/YouHb29tyfX092LKsDXq0QXutL7CdNXrNmrNnEbGKN4qEGd55eiBh+NJEPYJ9VURrJKzy1YW5\nrEVlKWUrEtZRRbUewlk6YmyCMpHw6YGE4csxFgGbLLOFuSwS1h10vk2lX5jLBnbqpIzX19fmnLAd\nUUoDAZ8j8bfzAAAPRElEQVQ2SBi+LFn0q1uV/Q65WiQc9ZSoRcK1UUXRxOSoAsJHw9FCn95DxKcH\nEoYvRVQJkQlYd8nVStQsEvbtLce6o0Vj6300HEXCWTpivV6n05MR8OmChOHLUZsZ56doZCVqUSSs\nndB8Z7RaJJylI3YZYW/X0caP6BpOByQMX5Ix+WYlarVIOJs/588aldYW5qLpybXKCBtjD78XSBi+\nFCZDTR3oWa9vbm7Kt2/fyu3t7aAWWDdi+C3J0aKfoWkBjXizqFbnxvkewSZjrQOG3xMkDF8KrXrI\nFtLsuLm5Kd+/fy/fvn0LJRwtukVd0Dy6COenJfvz4+PjpmG77xNsEmY78u8NEoYvhdb+WjSrGyv0\n+ubmpvzxxx+baDjbkuwFPCbjqDWltZ7UScnaptJHw0zLAAMJw5dBd8JFXdH89fX19UbAlpLQacl+\nO3LUia2UfwQ8mUzCcjRtTWnS1cGdy+Wy3N3dbUbZEwmDBwnD0aNy1N7AJmFbZNNjsVhsKh98fwif\njvAtKhUVsGES1sU2jYJtUoZFvz4SVgnTqB2QMBw1PjqNImEtN4u6omnLykzCfgFO8Y9rOWFt0O7n\nxpEThggkDEdLlB6YTCaDhjzallKrIW5vb7emYejh21TqJgy91rNdR+kIzQGbiO/u7jZz4zRXbBIm\nJwylIGE4UjIBR+kISz3c3t6Wb9++bRbjrq+vB8149BzlhJUsGm6ZGWcR8N3dXbm7uxvMjNPZcUTC\nUAoShiMkE7CdNR0xm80Gmy6+fftWvn//Xr5//16urq62Rtvr4XPCWeTr+/hm1RF+fL1K2M+NIycM\nBhKGo2dsYc5ywre3t+WPP/4o379/L3/++We5urraiDY7RznhUuL64Kg6QttQRpHwz58/wx1zjLEH\nAwnDUREtxPnHWiesOWGNhP/3v/+Vq6urQQ+I2qH4iggfDUfVEVqiZhURFgn7zmp6kBMGJAxHQ21e\nnJ11a7LfNed7RFxeXg6a70TNeKJtyb6Prz9bFYQuuNmimx06PdlPylD50h0NkDAcBVFHtOi+by+Z\nHSrbqC9EtDnD9wT24rRrvxVZqx58+Vk2rBMBg4GEoTu15uz+XtSop0XANRmX8l/KwY8kiq51Q4ZG\nxFb14Jvz1MYUIWBAwnA0+LaUURvJrGNaJOSafP3rZqVn0YRki4T9Jgy/EUNL0PywTkQMBhKGoyDq\nC5w1aa+lI3zuNxJ5JONa/a8/sqY8WTrCy9cLGBH/3iBhOAqilISPZrMouJYX9uKNOqYZ2U44f1gt\ncJSOeHp6GkTBPgL2jeABkDB0xy/CefH6s4+EvYx1PFEk9ywS1pxw1BNCG/RkkbAf5Pny8jJoBu/P\nRMGAhOEoyMYU+XlvXr61tEQ2KaNWHfH29pbuhNO+ELVI2ETs64CzoxTmxv3OIGE4GnzawIs1moDc\nsgkjqjn292o5Yd+i0hblWnPC0cBOBAwGEoajwAs4Glvfkgf20bA9t75OdB2VqGlOWDujqXxr1REm\nYXt+fa3oDL8nSBi6Ey2cjYm4FhHr39uFWiSsjXnu7+8HTdztiHLCr6+vm+dHthCBhOHTyLqjqWT9\nNmQ9ptPpYEKG9QX2vYE1Co6izuieNuHx8+Kio1YX7DdlANRAwnBQxnpB2FZkbTFpUvXHbDbbdEqz\nwZ06M04Hdxq1ygQtG1utVps0gx2/ImDkC60gYTgYWVWCf6yTMrIm7Ha2Cco2uPP6+jodV6SibW3K\n449Mwpp6sEqIqBoCYAwkDAdlrE5XI+HZbLaZlKyjiPSxTdDw05NNwj4VYYtt2own6mimi2x+arIX\n8OPj42AbM2kI+BWQMByELOrV2l27jmbGZYcN74ymJ0cz4zQStmhVm/HY2acjagJerVaDRu3ZDjlE\nDC0gYTgYURe0qDGP9gG2qPfq6mowNVmlG423z9IRWnZmO+H8lAtrypOlIvyGjNVqtSV0ImHYFyQM\nByUTsG/Ko5GwjrCP0g/z+Xzz5+zaS9jQSNhPwtCtyZGEs7ywSThKcdAnGHYFCcPByPoE+8Y8mhPO\nRth/+/atfPv2bassza7tsU7NMBn6VIR2RLNFNt0BN7Yot1qtBlGv7xVMcx7YBSQMB6HWlMc35/ES\n1nSElaTZsVgswhpiPz3Z8Dlh35jHxBpFv7WccNSeMmpVCTAGEoaDMdYdTSUcDe5UCdsE5dlstrWF\nWXfO6cKcDubUCRm+J4T1hfDRcCZik7Af/kmPYNgHJAwHJUtH6GGRbJQT/vbtW/njjz/Kn3/+uZFw\nNrbIN4TXpjx+RL3uivNd0cbywqvVqpRS7wGBgKEVJAwHQSPfrO+DPda8ri20RYftjIvK3fw9w3dH\ny6Zm+CnJfjeclqFZUx6AjwAJw0HQ6RjReHp9rNuPtebX/pwfUd/aH7iUMkgX+KhYc8ReuH40PSkG\nOBRIGA6Cph5MuFYL7K+9gCMJ10ScMSZgE67vflYbVQ/w0SBhOAg+/6vi9SVmJmETsQraomitMbbn\nr40rMlTEKmFdpItGEkW734iE4RAgYTgYPhLWCghtymM73/zGC5+OiIZ0evFGjdq1dMwEG421r7Wk\nJBKGQ4GE4SDowpymI1TCFv3W0hHWlMcP7bTXsPPYxIxaOiKLhKOtyIgYPhokDAfB54SzDRl6ztIR\ntUg4OitRTthvYY4i4SgKRsBwCJAwHITawpy1rLR64Ovr6+rCnG7AiCSs11EU7EUcRcJPT09hTph0\nBBwaJAwHIZOwj4S1M9pYiZqPhO11omslqo6oRcK1VAQiho8GCcNB0Dphq47wC3O+HWWtRM1XR/jX\nyogau4/lhFXEvh8EwEeDhOEg7BoJj+WEVcStRNUR0c45ImHoCRKGvaiVhulOuag+WHPCKuDaRo3W\nDRqeKC/sF+lUuFFLSuQLhwQJw060zIwrpQwGdLb0hsjK0vaJgAG+EkgYmvFjiWpnL2C/QSMScFQR\nES3GAZwSSBiayfoCR60kW+Srh6UrEDH8biBh2AnfEyLqD6wSVhlHItYhnZqOiErTAE4RJAzN+EjY\npKsLaHbtxVuLhFW+PgomJwynDhKGZjIBR8dYBOwlHD0HqQj4HUDCsBPRxIyoYXu2KOcb+JiEo6ia\nSBh+B5AwNBPNiVMBa1rB54SzaHixWAyiXp9rzrYrA5wKSBiaqUXBfmJGLf3gD2varsL1jxEwnCpI\nGHYiywvrtmS/O24sJzydTps3gQCcGkgYmlEhRgtzuj05GmWUHbYt2V6jdj7U9uHa1mS2LMMhQcLQ\nTLZZQ0Xsh3hG5Wa1ycl6biWTpPZ/GDu0v0TUrAcRw6FAwrATUTpCW1X6wwSciVifN3qt7GOllFSS\ndq0yHTu8fP3hXwPgo0DC0IxGrlGtcFQlEbWjjKJhfQ09Z3gxRqLcNRKOomDEC4cGCcNO7JuOiHbA\n1aZk1IjEm9371UjYPz/AR4OEoZkoEtZ0QxYF+w0YPh0x1pu4RiRLe7xPTjgTMcChQMKwE75b2r7p\nCG17adRk7Mmi1V3yvFnDd4QMnwkShmbGcsI+HRH1gmiJhLN7Ga1ybY2Ea/PkEDF8NEgYdqKlOqI1\nEo42YeySFx6LfndNO9Sia4BDgYShmbE64axEraVWeBd8OdquC3C75Iaj1wT4SLbnhwNUGEtH6E64\nls0au762MpYP3nchjlwwfCZEwtCMzwlrp7Nab+GoRngfEY9J1t9/fn4ejLDXqcp+wnKtZlhfH+Cj\nQcJwdHjZ6eNIonbt7z08PJQfP35sjru7u3J/f1+Wy2V5fHwsq9WqrNfrjawzKdcW6gB+FSQMR4Om\nF7Lz6+vrRppjx3K5LD9//twcJuGHh4eNhJ+enjbRshd5lLoA+GiQMBwF0e606Prl5aU8Pz+Xp6en\nsl6vy3q9Lk9PT5vHel4ul+X+/r7c3d1tBJxFwll6Iqq6APhIkDAcDS2lYhYJr9frslqttg6T62q1\nKsvlsjw8PAwOjYRN3irht7e3NE8McAiQMBwNLVUKXsIW1S6Xy8G13vPH4+NjGAmbgE28dk06Ag4J\nEobuRFUI2eYLTUdY5KsRrh7L5XIQHftzJOFaSRvAIUDCcDRku9/0bJGwSdjnfS33e3d3t0k52AJc\ndthCXvaa1AzDIUHCcBT4/G9U96sLc5qOeHh4KHd3d5sqiB8/fpSfP3+Wx8fHTaRri3j+sdYSs2kD\neoCE4Who2fWmkbDldlXCf//99+Z4fHzcbNLQTRvR9evra7UqAwHDoUDCcDS09HvIIuH7+/vy48eP\n8vfff5e//vqr/N///V9ZrVaDsrPa8fb2tvkc9POJrgE+EiQMO1Hr15ClEaJoVo9SSliV4K9tK7KW\np2k64u7ubiDi1WqVpjX85wTQCyQMzWg0+vLystkUYT0itEewRqy2AGaVCcvlsiwWi82hz53J1477\n+/vy119/lb///rv8+PFjawecr3TIFteIbOFYQMLQjK9QsNyszo0rpWylDUzA8/m8LBaLMp/PB4f9\nnbEI+u3tbdAPwrYi6+aL9Xq9kXBU5WD/Dj0D9AQJQzO2WUIjYS9gE54J2gQ8m83KbDYr8/l867qU\nshW5ZtfL5XKzCGcpCI2ETcLRRotsNx5AT5AwNOMj4UzAmqowAVuP4eha/+7YebVabXpA6AYNjYSf\nn5+rfYPt9RAwHANIGJrxu9YiAWsJmY46ys4XFxebvx/tkvOPbTEu2orsI2HqfeErgIShGa1seHl5\nCSPg5+fnzZBPm6yh5+iePUdWaaGHVUbo1mOrF9ZmPD4Sjl4D4BhAwtCMyjYS8MXFxSYKtokaOl0j\nu7bnqZW/2X3NNWvlhd7TnHD23P4aoBdIGJrRlIM9tsj4/Px8kyf2h5/MrI/Pzs4Gz2/n7Nqi7dqh\n1RFZRQTyhWMBCUMzJl1NF5ydnZWXl5etmXOWL/Zz5aL79tz6OtG5lPbxRn4bsn9ufw3QCyQMzfhK\nBR36qUNAx45SytZj/zrZ47EStqguuPbcAL1BwtAMOVSAj+ds/I8AAMChQMIAAB1BwgAAHUHCAAAd\nQcIAAB2ZsNoNANAPImEAgI4gYQCAjiBhAICOIGEAgI4gYQCAjiBhAICOIGEAgI4gYQCAjiBhAICO\nIGEAgI4gYQCAjiBhAICOIGEAgI4gYQCAjiBhAICOIGEAgI4gYQCAjiBhAICOIGEAgI4gYQCAjiBh\nAICOIGEAgI4gYQCAjiBhAICOIGEAgI4gYQCAjiBhAICOIGEAgI4gYQCAjvw/HJN2SfxqPgkAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc0c87b2110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classified as 7 with probability 0.999152\n"
     ]
    }
   ],
   "source": [
    "# Output may vary\n",
    "plt.imshow(val_img[0], cmap='Greys_r')\n",
    "plt.axis('off')\n",
    "plt.show()\n",
    "prob = model.predict(val_img[0:1].astype(np.float32)/255)[0]\n",
    "assert max(prob) > 0.99, \"Low prediction accuracy.\"\n",
    "print 'Classified as %d with probability %f' % (prob.argmax(), max(prob))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also evaluate the accuracy given a data iterator. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 96.830000%\n"
     ]
    }
   ],
   "source": [
    "# Output may vary\n",
    "valid_acc = model.score(val_iter)\n",
    "print 'Validation accuracy: %f%%' % (valid_acc *100,)\n",
    "assert valid_acc > 0.95, \"Low validation accuracy.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even more, we can recognizes the digit written on the below box. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">canvas { border: 1px solid black; }</style><div id=\"board\"><canvas id=\"myCanvas\" width=\"100px\" height=\"100px\">Sorry, your browser doesn't support canvas technology.</canvas><p><button id=\"classify\" onclick=\"classify()\">Classify</button><button id=\"clear\" onclick=\"myClear()\">Clear</button>Result: <input type=\"text\" id=\"result_output\" size=\"5\" value=\"\"></p></div><script type=\"text/JavaScript\" src=\"https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js?ver=1.4.2\"></script><script type=\"text/javascript\">function init() {var myCanvas = document.getElementById(\"myCanvas\");var curColor = $('#selectColor option:selected').val();if(myCanvas){var isDown = false;var ctx = myCanvas.getContext(\"2d\");var canvasX, canvasY;ctx.lineWidth = 8;$(myCanvas).mousedown(function(e){isDown = true;ctx.beginPath();var parentOffset = $(this).parent().offset(); canvasX = e.pageX - parentOffset.left;canvasY = e.pageY - parentOffset.top;ctx.moveTo(canvasX, canvasY);}).mousemove(function(e){if(isDown != false) {var parentOffset = $(this).parent().offset(); canvasX = e.pageX - parentOffset.left;canvasY = e.pageY - parentOffset.top;ctx.lineTo(canvasX, canvasY);ctx.strokeStyle = curColor;ctx.stroke();}}).mouseup(function(e){isDown = false;ctx.closePath();});}$('#selectColor').change(function () {curColor = $('#selectColor option:selected').val();});}init();function handle_output(out) {document.getElementById(\"result_output\").value = out.content.data[\"text/plain\"];}function classify() {var kernel = IPython.notebook.kernel;var myCanvas = document.getElementById(\"myCanvas\");data = myCanvas.toDataURL('image/png');document.getElementById(\"result_output\").value = \"\";kernel.execute(\"classify('\" + data +\"')\",  { 'iopub' : {'output' : handle_output}}, {silent:false});}function myClear() {var myCanvas = document.getElementById(\"myCanvas\");myCanvas.getContext(\"2d\").clearRect(0, 0, myCanvas.width, myCanvas.height);}</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import HTML\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "def classify(img):\n",
    "    img = img[len('data:image/png;base64,'):].decode('base64')\n",
    "    img = cv2.imdecode(np.fromstring(img, np.uint8), -1)\n",
    "    img = cv2.resize(img[:,:,3], (28,28))\n",
    "    img = img.astype(np.float32).reshape((1,1,28,28))/255.0\n",
    "    return model.predict(img)[0].argmax()\n",
    "\n",
    "'''\n",
    "To see the model in action, run the demo notebook at\n",
    "https://github.com/dmlc/mxnet-notebooks/blob/master/python/tutorials/mnist.ipynb.\n",
    "'''\n",
    "HTML(filename=\"mnist_demo.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolutional Neural Networks\n",
    "\n",
    "Note that the previous fully-connected layer simply reshapes the image into a vector during training. It ignores the spatial information that pixels are correlated on both horizontal and vertical dimensions. The convolutional layer aims to improve this drawback by using a more structural weight $W$. Instead of simply matrix-matrix multiplication, it uses 2-D convolution to obtain the output. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"https://thatindiandude.github.io/images/conv.png\" style=\"height: 75%; width: 75%;\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also have multiple feature maps, each with their own weight matrices, to capture different features: \n",
    "<img src=\"https://thatindiandude.github.io/images/filters.png\" style=\"height: 75%; width: 75%;\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Besides the convolutional layer, another major change of the convolutional neural network is the adding of pooling layers. A pooling layer reduce a $n\\times m$ (often called kernal size) image patch into a single value to make the network less sensitive to the spatial location.\n",
    "\n",
    "<img src=\"https://thatindiandude.github.io/images/pooling.png\" style=\"height: 75%; width: 75%;\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = mx.symbol.Variable('data')\n",
    "# first conv layer\n",
    "conv1 = mx.sym.Convolution(data=data, kernel=(5,5), num_filter=20)\n",
    "tanh1 = mx.sym.Activation(data=conv1, act_type=\"tanh\")\n",
    "pool1 = mx.sym.Pooling(data=tanh1, pool_type=\"max\", kernel=(2,2), stride=(2,2))\n",
    "# second conv layer\n",
    "conv2 = mx.sym.Convolution(data=pool1, kernel=(5,5), num_filter=50)\n",
    "tanh2 = mx.sym.Activation(data=conv2, act_type=\"tanh\")\n",
    "pool2 = mx.sym.Pooling(data=tanh2, pool_type=\"max\", kernel=(2,2), stride=(2,2))\n",
    "# first fullc layer\n",
    "flatten = mx.sym.Flatten(data=pool2)\n",
    "fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=500)\n",
    "tanh3 = mx.sym.Activation(data=fc1, act_type=\"tanh\")\n",
    "# second fullc\n",
    "fc2 = mx.sym.FullyConnected(data=tanh3, num_hidden=10)\n",
    "# softmax loss\n",
    "lenet = mx.sym.SoftmaxOutput(data=fc2, name='softmax')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that LeNet is more complex than the previous multilayer perceptron, so we use GPU instead of CPU for training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Start training with [gpu(0)]\n",
      "INFO:root:Epoch[0] Batch [200]\tSpeed: 19760.87 samples/sec\tTrain-accuracy=0.111300\n",
      "INFO:root:Epoch[0] Batch [400]\tSpeed: 23068.24 samples/sec\tTrain-accuracy=0.112450\n",
      "INFO:root:Epoch[0] Batch [600]\tSpeed: 23056.69 samples/sec\tTrain-accuracy=0.112700\n",
      "INFO:root:Epoch[0] Resetting Data Iterator\n",
      "INFO:root:Epoch[0] Time cost=3.010\n",
      "INFO:root:Epoch[0] Validation-accuracy=0.113500\n",
      "INFO:root:Epoch[1] Batch [200]\tSpeed: 23184.43 samples/sec\tTrain-accuracy=0.145950\n",
      "INFO:root:Epoch[1] Batch [400]\tSpeed: 23032.03 samples/sec\tTrain-accuracy=0.775550\n",
      "INFO:root:Epoch[1] Batch [600]\tSpeed: 23030.07 samples/sec\tTrain-accuracy=0.912350\n",
      "INFO:root:Epoch[1] Resetting Data Iterator\n",
      "INFO:root:Epoch[1] Time cost=2.605\n",
      "INFO:root:Epoch[1] Validation-accuracy=0.934400\n",
      "INFO:root:Epoch[2] Batch [200]\tSpeed: 23112.07 samples/sec\tTrain-accuracy=0.943700\n",
      "INFO:root:Epoch[2] Batch [400]\tSpeed: 23030.54 samples/sec\tTrain-accuracy=0.954150\n",
      "INFO:root:Epoch[2] Batch [600]\tSpeed: 23102.48 samples/sec\tTrain-accuracy=0.964900\n",
      "INFO:root:Epoch[2] Resetting Data Iterator\n",
      "INFO:root:Epoch[2] Time cost=2.605\n",
      "INFO:root:Epoch[2] Validation-accuracy=0.972100\n",
      "INFO:root:Epoch[3] Batch [200]\tSpeed: 23115.03 samples/sec\tTrain-accuracy=0.973150\n",
      "INFO:root:Epoch[3] Batch [400]\tSpeed: 23055.29 samples/sec\tTrain-accuracy=0.973600\n",
      "INFO:root:Epoch[3] Batch [600]\tSpeed: 23091.43 samples/sec\tTrain-accuracy=0.976350\n",
      "INFO:root:Epoch[3] Resetting Data Iterator\n",
      "INFO:root:Epoch[3] Time cost=2.604\n",
      "INFO:root:Epoch[3] Validation-accuracy=0.981000\n",
      "INFO:root:Epoch[4] Batch [200]\tSpeed: 23060.63 samples/sec\tTrain-accuracy=0.981000\n",
      "INFO:root:Epoch[4] Batch [400]\tSpeed: 22992.31 samples/sec\tTrain-accuracy=0.980500\n",
      "INFO:root:Epoch[4] Batch [600]\tSpeed: 22896.53 samples/sec\tTrain-accuracy=0.982650\n",
      "INFO:root:Epoch[4] Resetting Data Iterator\n",
      "INFO:root:Epoch[4] Time cost=2.616\n",
      "INFO:root:Epoch[4] Validation-accuracy=0.984000\n",
      "INFO:root:Epoch[5] Batch [200]\tSpeed: 23072.58 samples/sec\tTrain-accuracy=0.985200\n",
      "INFO:root:Epoch[5] Batch [400]\tSpeed: 22997.41 samples/sec\tTrain-accuracy=0.984050\n",
      "INFO:root:Epoch[5] Batch [600]\tSpeed: 22970.29 samples/sec\tTrain-accuracy=0.985950\n",
      "INFO:root:Epoch[5] Resetting Data Iterator\n",
      "INFO:root:Epoch[5] Time cost=2.613\n",
      "INFO:root:Epoch[5] Validation-accuracy=0.986000\n",
      "INFO:root:Epoch[6] Batch [200]\tSpeed: 23238.82 samples/sec\tTrain-accuracy=0.988150\n",
      "INFO:root:Epoch[6] Batch [400]\tSpeed: 23112.63 samples/sec\tTrain-accuracy=0.986750\n",
      "INFO:root:Epoch[6] Batch [600]\tSpeed: 23082.98 samples/sec\tTrain-accuracy=0.988100\n",
      "INFO:root:Epoch[6] Resetting Data Iterator\n",
      "INFO:root:Epoch[6] Time cost=2.598\n",
      "INFO:root:Epoch[6] Validation-accuracy=0.986700\n",
      "INFO:root:Epoch[7] Batch [200]\tSpeed: 23132.00 samples/sec\tTrain-accuracy=0.989550\n",
      "INFO:root:Epoch[7] Batch [400]\tSpeed: 23017.32 samples/sec\tTrain-accuracy=0.989050\n",
      "INFO:root:Epoch[7] Batch [600]\tSpeed: 22990.25 samples/sec\tTrain-accuracy=0.989900\n",
      "INFO:root:Epoch[7] Resetting Data Iterator\n",
      "INFO:root:Epoch[7] Time cost=2.609\n",
      "INFO:root:Epoch[7] Validation-accuracy=0.986700\n",
      "INFO:root:Epoch[8] Batch [200]\tSpeed: 23114.52 samples/sec\tTrain-accuracy=0.991400\n",
      "INFO:root:Epoch[8] Batch [400]\tSpeed: 23052.39 samples/sec\tTrain-accuracy=0.990800\n",
      "INFO:root:Epoch[8] Batch [600]\tSpeed: 23076.56 samples/sec\tTrain-accuracy=0.991150\n",
      "INFO:root:Epoch[8] Resetting Data Iterator\n",
      "INFO:root:Epoch[8] Time cost=2.606\n",
      "INFO:root:Epoch[8] Validation-accuracy=0.987400\n",
      "INFO:root:Epoch[9] Batch [200]\tSpeed: 23134.78 samples/sec\tTrain-accuracy=0.992750\n",
      "INFO:root:Epoch[9] Batch [400]\tSpeed: 23071.56 samples/sec\tTrain-accuracy=0.991850\n",
      "INFO:root:Epoch[9] Batch [600]\tSpeed: 23073.45 samples/sec\tTrain-accuracy=0.992000\n",
      "INFO:root:Epoch[9] Resetting Data Iterator\n",
      "INFO:root:Epoch[9] Time cost=2.604\n",
      "INFO:root:Epoch[9] Validation-accuracy=0.987400\n"
     ]
    }
   ],
   "source": [
    "# Output may vary\n",
    "model = mx.model.FeedForward(\n",
    "    ctx = mx.gpu(0),     # use GPU 0 for training, others are same as before\n",
    "    symbol = lenet,       \n",
    "    num_epoch = 10,     \n",
    "    learning_rate = 0.1)\n",
    "model.fit(\n",
    "    X=train_iter,  \n",
    "    eval_data=val_iter, \n",
    "    batch_end_callback = mx.callback.Speedometer(batch_size, 200)\n",
    ") \n",
    "assert model.score(val_iter) > 0.98, \"Low validation accuracy.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that, with the same hyper-parameters, LeNet achieves 98.7% validation accuracy, which improves on the previous multilayer perceptron accuracy of 96.6%."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "After training is done on the CNN model, like we did with the perceptron, we can now predict a single image. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV8AAAFfCAYAAADptc+BAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnWtXIkmzRhO8gZeeXvP//+NMtwqCIOfDe4J5CCIyExss\nxb3XqlVF2Q22rZswMjKiFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+CaMBn79zcCvDwBw\nCppuHX/EZwEAALsgXwCAAUC+AAADgHwBAAYA+QIADADyBQAYAOQLADAAyBcAYACQLwDAACBfAIAB\nQL4AAAOAfAEABgD5AgAMAPIFABgA5AsAMADIFwBgAJAvAMAAIF8AgAFAvgAAA4B8AQAGAPkCAAwA\n8gUAGADkCwAwAMgXAGAAkC8AwAAgXwCAAUC+AAADgHwBAAYA+QIADADyBQAYAOQLADAAyBcAYACQ\nLwDAACBfAIABQL4AAAOAfAEABgD5AgAMAPIFABgA5AsAMADIFwBgAJAvAMAAIF8AgAFAvgAAA4B8\nAQAGAPkCAAwA8gUAGADkCwAwAMgXAGAAkC8AwAAgXwCAAUC+AAADgHwBAAYA+QIADADyBQAYAOQL\nADAAyBcAYACQLwDAACBfAIABQL4AAAOAfAEABgD5AgAMAPIFABgA5AsAMADIFwBgAJAvAMAAIF8A\ngAFAvgAAA4B8AQAGAPkCAAwA8gUAGIDLoT+B78poNHrXvdbjFpvN5qA/3/r7rec79PX+9PMD+Cog\n3wEYj8dlNBq9+8j+fil9cjxUmJvN5qAjep7ounYv+7sA5wLy/WC8QMfjcXq0Pu6PHqn56+ixv//2\n9rY9NpvNzmP/MRWwv47uZY9Ho9GezAHOCeQ7ACrVi4uLcnFxkV7X7vmP98hOz7Vrfbxer8vb21tZ\nr9fN67e3t53X/ZMj+pwAzgXk+8Fo5HtxcVEuLy+bZztqjy8uLlJ5ZULrTQusVquyXq+7zv71fETs\n70XXKnCLgAHODeQ7ACrfi4uLcnV1tZVodH11dZVe29nkW0p/1Gm0pLxarcrr62tZrVbNay/R6Kyp\ni/F4vBMx6z0TLwKGcwT5DoCmHFS019fXW7naYfdq5+vr6zDyzSLPWhQcnV9fX8vr62tZLpfba38s\nl8sdkapgfZ7YP9YFw1LKzj3EC+cK8v1gorSDSjQ6bm5uqtc3Nzfl8vKy+9f6moCje8vlsnpYzlnl\n23Os1+u0VM4iXoBzBfkOgE87qIBVqDc3N2UymWyva4+vrq7SX/Wje6XElQfR9WKx2B7L5XJ7bblm\nk+9oNNpZeDPBRtcW3a7X652vjb22iRcBw7mCfAcginxVvibV7JhOp3uPr66uDvp1v5R6qZc+fnl5\nKYvFory8vJSXl5dtrtmLt5SyVwVxcXGxfTwej3c+FqFROuKFcwb5fjCadhiPx9ucr6YSVLDT6bR5\n3N7eluvr67T2NrtfSrw45+/P5/OtdGvitcU5LT9br9fbdIT9my3doHL10vWbRwDODeQ7AF68PupV\nqfpzdH13d5fKt2dDRK00bbPZbN8YVL4mYN0QMhqNtmVnPcdqtdrKeDwel9VqtbeDz7BoHT6GaJGz\ntfDJwuhhIN8PxqSiZWYa8ZpU7+7uyu3t7V70a6kGXXgzGfbuQqvtRovuaaTsN4j4Nw+Tr49+s3tW\npubrhf01P9h/TkuoPYu0+lj/fm3NAGKQ7wBk4jLB3t3dbY8o52sRskWjKt8s5+uve3K9eu2rNPSN\nQ6N2law/+3uHbt6A9+EFm92rLZD660zI2W9UsA/y/WB8vtdHvpZSuL+/L3d3d3tVDT7q1VRAa1ND\nLeq1cyZkv0ioeerr6+uyXC7LZDLp+gH2Qvayje7xQ/w+ov9jO/t7ta+/v9fzRm9v2vzfxSDfAbC0\nQyRfSznc3d2V+/v7najSl6JlkW/Pr4+1H0L/Md+HwoS/XC7L9fV1mUwm25rfrMwsq/ONfrijx/wA\nv4/e1JItluquRX+2a61g0f9j++1IF1T5f8tBvgPg0w4a3Wrka/JtbcIwAff8CliLerOzz/GuVqty\nfX0d7nQ7pOIiW4SL7vND/D56vxc2m/2djPaGqvfse8H+n2zBVDfMaKoKcpDvB5PtblPxWtR7f3+/\nt93YbzvWyPdPpFu7p5+rRUA+IrLjkLxzb1UE8n0/tTdj/zHbQOPPtqFmsVjs5P5NwFrp4p8fcpDv\nAKiAs5yvCThrtuOvvXxLaW+iMCIB67VFu7UIVascsjyzP9eqIJDvcehJQ9m1baTRDTVaVqjlf74v\ntb2Wndkk0wb5fjBZ5Kvy1cjXt42MWkleXl6W8fh/4/ha+Vu9Z0Ri03vr9bpcXV01F8+yRb3onk89\ntErTkO/76HkTtPPLy0uZz+dlPp/vbahR8Wp+16caLE3FBpk2yHcAoqqBqM7X5Ftrpm73Tb6lHD7J\nonU/WsVulbD1HJl8o2vk+z58vj37v3t7eysvLy97C7m6iaaU/8rRotfRiNe+H1l0y0G+H4xussjq\nfHXBrTY2yO8w87Qi2h6ilEVvWqP22Ee/WS2wRtRwOD0Ln3bMZrMd8er2cX2z9HXX/v+SyLcP5DsA\nKk/79U63Fmv067fbRtdR3u1Pee8PTi3t4c9ZCgP5Ho+abP1h3fGinh0qXpWv/23IytCQbxvkOwD6\njWzlPJZve35+3m4jjgZt1q7fk26IWjdG173nHjKR+gnNev8YUfxXksGx/m369ayJd7PZ7JU03tzc\n7FS0aD6+lLL3G5d9X3t5Qwzy/WB8FPH6+rpdXZ7P52U2m5Wnp6dyc3NTSinVtIOfdBz9iq/X+tiv\nWvc87jmMmsDt2j4f/xwqXv9n3yNhL4HPKIXav6H2sejf4r/OOpapNm3Edipqn45MvtFim1a+kHpo\ng3wHwEe+Jt/ZbLazfbiUfflqnlcfq3xblQYmX/0ByY5amiO7VxO33Te8gLX9pN23etEsio+uD3kj\nGJpW9UntN4XoWh/r17NV7WCRr26amUwmYemfoqmi1WpF5NsJ8v1gNPK1b3CNfJ+fn7edyqxsx1c1\nRGddFPHSjUrAWhL1aY3ocfaxTN5RVKuoMOxz1HrRVh5Zr2vplM8g4J6Kkz9JGenXzL43VMTRG7PK\nN+o2pzl5e25dKF2tVnsLdZCDfAcginzn8/lOmc/FxcV2ASMrN9PHWU4vWuG2H8SaUP19jbhr1/7v\n2Q+7/jD6HHX2q7OKo5S+PhQ9uekh5duTXmjl7kvpf3Ox/2v7PlDx+sPEe319HaYaVL5+wdRSFMi3\nH+T7wWh+zOd8Z7PZzjevLV7o5IjsMNFFzWyie1EKo5bWqL0J+EjcX9u/OcpNK14aPqfYe63P18pl\nfxStvPQhVSLRG0h2T+Waidee13p21MSrEbOvgNAp1uR82yDfAdC0g1Y7+B1FJl8v4OjaZB2VaUW7\nxVppjEiy/k2g9qagUbuKx34gW5FclvfMxOHz2fpaLQl/BD3ybUX1dp29iUT/tuzrFD1W+WbRrspX\nAwiLev2EE8hBvh+MfeNatGBpB/vG1ehQc2jZtmIv355+CT6dUdtBV3vN6Np+UE26m83/GvMYkRDt\nvhLJuCVd/Vi02DdU9HtI1NtzXXtD8fdaItezyfT6+npPvCpf/bOWplgulztvxkS9bZDvBxNFDYvF\nYuebtpT/RccaTUSHSs/Lt9YnN0pn1KJa38BHG/us1+sd6fq+woYuuLWi3tavz61DX6/nOPX/d8/H\neqNTL9+ahP1vGNnrGev1eiveaJ3Ay1d/c/O9IEg7tEG+A+DTDtk2Tosmsi5mes/kG80/89cmzN6o\nVn+w9Nqex8bWq4Az+eqij93Xj+u1l4g/ohK6TL61srpTkwnYy7AW0ev997yxRJ+Dv9eKdo3RaLSz\nQWixWOyJlwW3Nsj3g/GRry1QaMRrH9Nv6uxs1+PxeK+3bnaYNGuHf3477Dks6r26ugpFYaj8vCyj\nVEN0+IUiex6Tuf9YS7xRTfIp/7977rcief91y75Of/LG0op2DZXvYrHYa+pPtUMfyHcAVMCafvC/\nqllu+PLysry+vparq6s0FdEb+VrDc41ua5GvF3DruidK1885qhGOzirXqIROH6uMaq/xkQtCkWyV\nKIqP3lg2m0213NBfK70y9K+plQ1Z6VktUoYY5DsA+o0d7QpS+aoQ/Yqyimw0GjXzvdrwvLeKIsr5\n9og1ErvVL0er4q1Dv15RPbMKwos3K6v7qOislXe1/+vozSU6134D8qmoWuoh+rfXxKvfr7Xa3yxi\nhl2Q7wfjv7HtG/j19XVHvPZnTMwtUap8eyoeWnXDUbVDJP3eCDr6c1mpW3TP/7ZQK6UbjUbdZXQf\nmXaIFrnsOnsz8RJ8e3vbGaCqhy2YlVK2Ub3Pu/vPzcs5irqjyFffyHtSFbAP8h0AFYnlar14N5vN\njiRrpWAmkUhEmbAy+WXXPRGyL4mrnWtH9G/NfvW1NIreK6V0v7GcMvVQk60/926OeXt7K5PJZHtY\nE36r3y7lf+K1Rc9S9munvXCzCLgn5VCrBUbAdZDvAPi0g/9hsPv+V/NaNGfyzX5w/ePabrbouvUm\noPcjwWaS7n1sb0a+30DUf6CU0iV/+9X8lP/PvedWNK/3rN+z9X6OxKuLkpGEs4g/Sjv4z6Un7QBt\nkO8AZOItZb9DVCtnqbnL7NfU6FfD6O/Xnr/2JhBFz4emNFrXloLxi4dRJYelHVoVHaeUb020mXx7\n0kZvb/8b92Oj3O1eKf+J1ypRfMpBpRsJOKqsqEW+rbQDEq6DfD8Y/aa2PK1h6QBLN9QE6a+1Rrjn\n7KsJapUGvYLuiZR9hNxzWB2xH1Wvj/V6NBqFC4HR9akrHrxsMxlHEWV2b7FY7FSulLIr3pubm50I\nVMWbSdenvLI3cf9GF0k4+nfCPsh3AFS++ng8Hu+Jt6dWVRdXskUTf/RuQog+hx5R11Ikmkdu1THb\nWeVrvQT0Wu+NRqO9UrioPM7qo0/5/6z/L/7/SB+30ikqu9fX163o7Ouu4rU/q59HJOBMxP7Nuhb5\nZuJFum2Q7wBotKPitR8kPwlAF+OyAnt7Xn3+2g999Fy992pHFLFnueRIjtnx9va2/VVbj+ieydf6\n0+rz+McXF7u1sKf4f+75P4nSJ1laRcVqX1OrgLCvxyGRb/Z5ZwKO+vtG4kXAdZDvB2PfkBa16A9I\nJj27Nvy9qJZTv/FrPwy158o+Fn1edu5JS9gRCTG7VvnqOboejUbbv6u7r7Qsyx5/hHxr0vXy9amV\naJSPfs9cXFxsI97JZFKWy+Xen7M/25PzjX57yqpMetIOkIN8B0C/OXvrTI9Vj+p/IN9D7e/15Irt\ncSbE6PF6vd4KtnWYfLN6WD0uL0/zI1ATbXRoGqV2Xq1W26/z5eXl9t84nU7LYrHYyldz+/Z5tK71\nc/fi7V10i95sIAb5DkzvN+gpvpFP8ZyWQonSKT5H7D+X2g+9j3z9taYf/G8RtUWkU0W+NflG96Pc\nrkWvmqP3aZ0sX19KvoNNf+PS+z614Jvn6Lirl5eXrezta+/TEci3DvKFk+B/yFXGerYf+PF4XF5f\nX/eeQ4UZ5Xh1p5Wd7blNIl5K+vkMkfP1j6M3nFL+mzxsOxjt36W/GWgFh1aUqJD18/Gfn71OKbs9\nev0bm4nX5BsJWFMjpB/aIF84Ol68eh2JN4rWVEoWAfoqB78QpQtEJnQfFXrZDVlqFolYpeUrSey+\nzvrTTmImXi/d6HOKHmvUqwL2Ua+JV+UbvRki3jrIF05CJF7DFhsj+drf9Qs9vsY0qgDQ8qtoE4vK\n7SPk6wVbu2f4BU8vUc1n+2ZGPQLW1/LXmm7QyNdHvbPZbCfyVQEj336QL5yU2g+gydHXpL69/TcR\nw+/2qtW/9shXI2Mr6Tv1vz+TrZeuz99mNda6GBlFvl6+tdfX6yzyzdIOdo+c7/tAvnASoh88XTjS\nPKN+3C+GWd2zyqG24m5C8c+rz+13EZ7631+Trl1n/TqirdtR2iFq0Wn43Hst5aG/UdTEa/L1C57k\nfPtBvnB0TLJ2nf0Z/1irJCxna0IxofqFqagqwiJffW59Xi+4U34deq6tisHK3vRz0t2AdkRpB7+F\nO4p89WsRHb0539lsthPxar7XytzsgBzkC0elVsqU7bAy8dpCXFSa5qNXjXT9Pb9opxUWuoMwKnk7\nxdcje6x58aurq+21/Xs18tVdeT7yzean+f+H1gLfITnfxWKRLoCSdugD+cLJaIlYNwJoTbAedi8T\nRvRYpWsfj+phT53v9f/m7L5WelhpmX+svRv8RpRaqZl/bRVwtn3Yb9+Oqh20uY/fCk3aoQ/kC0en\nN+3gtytHZWdRnWqWs9Q/40vd/HPWNiIckx756JuBNY6PIl/dsddT45t9Hv6NS+XrI1/L6/rod7FY\nhHl3Ftz6Qb5wErIFN6UmvkPqVKOPvfe5h0AjXOtjYfe1/abJtyXg7A0m+u3AyzcTsE87LJfLvXRP\nlBKCHOQLg9Ej0c/43O8hE75POWirTe3bYFMrdIqFjRHKSs4snePlmh0mWL+Bwm+isEPl7a9JObRB\nvgAnJEpv+Osor2tdylS6d3d35e7urtzf35e7u7tye3u7J2AVbym7PRtqE0BWq1V5fn7e20BRKyPL\nGukg3T6QL8CJ8OKNctlRRcP19fV2QKbJ16R7f3+/E/1Op9OtfDX9kMnX9z7WfhmRfK2qwS+m9QoY\nEecgX4ATEi30+Xs+r2vphh75RqmHWuQbNcyx69lstiNfq+WNIt+swoR0Qz/IF+DEePFG5XSadohy\nvSbfh4eHrXRVvjqZw+9wi+Srka0dWdrBImRfyZAJFwn3gXwBTkhNuL7ETDdRZJHvw8PD9mP256JF\nt1bkqxUMVsWQpR2idpHRwhrCPQzkC3Aiog5lUbMcTTto5JvJtzaVo7Xg9vr6Gm4XzuSbtYu0crha\nzTXUQb4AJ6QW+er2ad+3oSbf2jRmL2C/ddhHvpbn1Xxv1Cy91jSHhbb3gXwBTkxLvFHkW6t20CY7\nJlvf2aw37WCR79PTUzXyjVp32vPr2V9DDvIFOAHRpopMwlGpmUW+WuNrka9upIjOtZyvVjlYhDub\nzcrz8/NBdb6+Y1nPjkbYBfkCnIhausEfrU0WKl/NF2fnQyJfE+/z8/NeysF3L4u2DiPZ94F8AU6E\nr2bIDi9cX8WgHcyur693nrt2lBKPZMomVfitxL68zFc5wJ+BfAFOhFUyZPlZXWTLtgv7RTR7Xi9Z\nJaq5zXo89MzEQ7inAfkCnIioWY4/Li8vt7ld3TShW4ajCRVZox5rRZn17fWtH6NevF68CPg0IF+A\nE1DbveYPla8J2Ee+0WDMrCexCrindWQ0iSLbzQbHA/kCnAjN9WoVg8/paueyKPLVXWu1JvC+ib0K\n2Eexvp1kJl6/wIaAjwfyBTgRtR69VserFQ0W9Vrkq20iff2uPb/HC9inHXpyvtEIeCLf44N8AU6E\nn1Dhd675xjkmYIt8s11rLWoLbl68rZxv1MsBjgPyBTgBtd1rUQ1vFvla2iGazdYjwlbagWqH4UC+\nACfCdyzLxGuHNkevRb5+Tp2Xoo98o7RDNDDT53yz9pFwHJAvwImo9ertjXw15+tLzFTCWWObKO2Q\n5X01FVGLfBHwcRi3/wgAvIesXWQU+fZUO2SlZp6oxWPvght1vh8HkS/AgdSGYRom26hNpF9081Gv\npRy8eHU6hZdqdp1NI462Eke1vlQ7nA7kC3AAPT0VRqNRuby83JOtlpbpKKBoFlutyiFKH2TXv3//\nLo+Pj9uWka22kT79wDj404F8ATrxEyhqZ98sJzq03ExFrSOBaj16/c606HiPfGt5XzgeyBfgALJe\nvL5FpMo3krCKdzqdbtMSutDmUw4mX4tso/Hv/vz79+8dAdf69VoXM416ST2cDuQLcAAqXt/UXB/7\nrcS19IPJ1xbkfNrBVzroWCBtCakDL+368fGxGfn6GW3ZFmM4LsgX4ACyHr0qSltky9IOurhm8m3N\nY/OLbX4gpjVG1wW2l5eX8vT0VJ6envYi3yzt4PPJ5HxPB/IF6ERLvDT69f15bbHNN0evpR2i54ha\nSZbyX9rBIl8/EshPJNao1yJfXwFhKYdaiRniPS7IF+AAMvFq5Kry7V1wiyZcRAIuJY98bRabn8nm\nzyZmTVOYfLUe2O+OQ8LHBfkCHECUdlABa8/e1mKbXkeLd36xzXaymRyzeWyWatA0gz9HaYdMtGwv\nPg3IF+AAVLzRlIpMvFnu145Dh2Jmka+NgbcKB5OsCldzw5F87TWiazgeyBfgAFqRr5dvS7x2rc9d\nuy4lz/laasEW2H79+rUj2ehaS80iybYew/tBvgCd+Brflnitblfrd/1EYista6GTKVqj4FXCWoKm\nOV6t7bUSM8T6sSBfgAPIIt8o7aBy9XW7PpVg+HaQ/nqz2eyMfveH9m3woo22DpNSGA7kC3AAfneb\nHwEfRb5Zk5yoRWQr52rbirUPb494fe8GNk8MD/IF6MTX+WaRr4q3FvlmXcpah2+A7rcV6y63qGtZ\n1LEMPh7kC3AAhyy2RX0aot682gy9lLhdpJ4PiXxrrSKR77AgX4ADiHo76JZgX+Xgo9+etEO00cGu\ndaEtE68KOGsVqQKGYUC+AAfgF9x6qh1UvFnawYgE7I8o6vUStsoGlS2R7+cC+QJ00io1i3K+mu9t\nVTvUot5o6GUt7WACrjVbZ8vwsCBfgAMwYfoeDBr5tgQcpR2ygZdemrWoN5Jw1izH55Lh40G+AAdQ\n62hWq3aIIl8/HigaeOn760bTKrJ872KxCEXrr2EYkC/AAUSbLGp9HbKZbF68pdTzvZGAa/leO9ea\n45ByGBbkC9BJNEKo1tHskJxvFO1GUa5KNhob5Gt6tZpBpatnGAbkC+DwEane9ymHrOJBxduztdgP\nxcxyua+vrzt9ebNpFD6XWxMuEh4G5Avw/0TS1Xs91Q6RgFtbi33EW9sqvFgstpMpanPYsh4RtXvw\nsSBfgLIvXi9dO/sNFrVa3yjqzXK+tU5ltnhm19n4d9+7wedzEe7nAvnCtycSbXT93sg3SzvY82q+\nVysYtO+uNkLPRsDrHLZaH14k/DlAvgABKkc7HxL5ttIOh0S+1qNXh2LqIEyf8601zUG8nwfkC/D/\nRML159b8NhWwr3SImuqU8l+lQyZfE69J14+Az+TrIff7uUC+AEImXLv289tqAzR1onGU740iX11w\n05SDHwGfpR2s2sHX80Yg3mFBvgAlX2CLFtt6xHt9fb0T8Wa723zFg498Lbq1yPfx8XEnBRHlfDXy\nRbyfF+QL8P9EC2x6neV9MwGrcGu721pz2XQm29PT084kYh2E6UvNSDN8bpAvgODTAe8V783Nzd7C\nnKYsok0WPUMxHx8f9yYR+1Iz7dOLbD8vyBe+NV6utUNF6pvq+EW3q6urHVH7KodDIl+V7/Pz845w\n/cQKSzvQs+Hzg3zh2+MX0jJpTqfTcnd3V25vb8vt7W2ZTCY7Eyui5jnRnDbfb2Gz2exMnHjveCAa\npH8tkC98a/xUCp+j1WuT793dXZlOp2U6nVanFGvUbOgU4s1mU0ajUbVPrzbO0SMaiJltsIDPCfKF\nb49vD2kpBH/OIt9oUGbWMtLOvszMTyPOJlP4Pg5Evl8X5AvfGs3n+l1qdm3n6XRa7u/vdyLfLO1w\ncXGRVhv4+7UevVHawbeZ9JEv0e/XAPnCt0fTDjqNwjdG95FvlnawyLfWuFwf904k1qoGHYjZ2lIM\nnxPkC98aLR/z8tUFtclkss33mnhbC261BTYd5ZOlHLLpFH60kJ/ThoC/BsgXvj1R2sGEa+kFi3ot\n7aACzsYFmQy1c1k0nTiaVlGLgFW20TXy/RogX/jW+GqHq6urHfFaiuH29jaMfGtz2krZX1zzAu6Z\nyRYNxfRz3rzU4fODfOHbk+V8VcBR1OsF7DuX9Yg3mtXWWnDLhmIylfhrgXzhW6Pbhi8vL3e2B08m\nkx3x3t/f70TCJt5atUOU883kW5vbpgLOBmEyGPNrgXzh2+F7N/gpxNfX1zv53tvb2618NQfcqvMd\nj8dlvV5vXysSry60RTvbovQDnAfIF86SqDdDdN/qd/3imh4+zaDCNWlHW4mjfg2RXP1cNr+Zgp1r\n5wnyhbPDotnWeTwe7+R2Taw1GWvdry2yRbPZMvlGE4lfXl72GqTbx7RTGfI9L5AvnCW+/WN2bZsn\nMtla1Gtn3YARLbJl8lXp+onEKl9tFRnNZIPzAfnC2RHNWouOy8vLnWi3J+3g57NpnjeaTJG1ifQT\niX3aQRvoIN/zBPnCWRJ1K/P9d02+ms9tCdj37LW0g59QkcnXxKsDMK1Xrx8NROR73iBfODuiqRNZ\ntzKf620dkcB70g61Bum1mWyticTwdUG+cJaoeKNRP3btpdsSsU9b+DFBvlVkbSimTaaoTSQm8j1f\nkC+cHVHO17eLtMNkW1tw0y5m0YQLP5utlN3I1yJYXWTzc9nsMWmH7wPyhbOkJl/dleaj3myhzQ6d\nwaala3pdSt9E4tlstp1IbEJmwe37gHzh7PA5X797TQV8SL53Op3ubNbQ1/P3WgtumnZ4enraKUEj\n8v0eIF84S3zkq9UJKt6eSgeLgieTSSml3VNBezq0Ftws8vU9e9lkcf4gX/hy+Nlo/r5fZFPZaioh\nahOpKQldULNoOppEER2t9pDREU0kJuo9X5AvfBmiX/Wj62wqRdaf189l813KfC7Xt27UfrpevlFz\nHFt400NFHYkXAZ8fyBe+HL5Rjj/rIpvmeHUihT808m1NIvZ9eaPm5tFIoFok3DORGM4L5AtfiqhD\nmV77yFcnU1jawaLeh4eHbcQbTaeIplJE44D8OJ+3t7fmFOKeicSMBjpvkC98CVS0+jg6dJFN63n9\nLDZNN2g3M92EkUW+Gu1qikD79Pb05j1kIjHyPS+QL3wpatJV+dqCmy8riyJf+5ies7RDFPlG04Qz\n8WYRcPQcpB3OG+QLXw4v22jjg5aXtSJf3yYySjtEka+mBVojgbKFNztHz0fa4bxBvvCl8DneaIdZ\nFvlqtYMLrxuqAAAPEklEQVRGvlGnsvdEvn4IZjQWKEs71BbviHzPE+QLX4ZsFJDuZosW3LSHQxT5\nZp3KdESQ0pKvilcrGWo1vj63mz2G8wH5wpciqnDw4vWlZq3I1zdd9499k/RD0w6+nCyKfvW5a9dw\nPiBf+JLUot9aqVkU+Wbz3XxaI9rdVks79JSaWdrBg2zPH+QLXwIf0fprPd/f35cfP36Uh4eHcn9/\nv1PDq6VkfvyP70ym1Ho1REMx5/N5eXp6Snv0WikZUe33BfnCl8AW0rLFMT3u7+/Lz58/QwGbeHUh\nTQ+PitHLV7cH+45k1jQnms2mbSKZTvF9Qb7wKfEiVPlqOZiOcrfH9/f35a+//ip//fXXNq9rXcm0\nhtfL16PirUW+1odXh2KqfKPxQBr5wvcE+cKnIutYpiOBogkU+vj+/r48PDxsI9+afK2Swb/uZrPZ\n5nntOhoFr03QLcLVRumtyBf5fl+QL3waMvHaxyzy1S5l2ptBKxlsQc1vIfYbKKII22hFvppy0GGY\n0Wy2KOeLfL83yBc+BS3x+sj35uYmbAtp+V3tVqZdy6Kcb4YXYzaZQiNfa47uh2Ja6oHIFwzkC58a\n3yrSbxe2el09tDevn9EWpR18hKtnvW7NZLORQI+Pj+Xp6WkrXXK+EIF8YXBakyns2stXN0v8+PFj\nu8im+V1/jhbcDC/CaCxQbSSQl6/NazPxMg4eFOQLnxIvZE07+Jzvw8ND+euvv8rPnz/Lz58/y+3t\n7c54eG2aY4fP+dqiml0bvodvz0y2x8fH8vj4mM5ls8gX+X5vkC98OqJFsFrka5sqfv78Wf7+++9y\ne3u73eGWnS3nOx6Pt7W2NenWphFr5Pv8/LyVbzTFgpwvGMgXPhVZ9YG2idScr5WVWeT7999/l+l0\nutObITu0zteIUg++l0Ot2kEjX99oJ5pWgXy/L8gXBqU1j83Ofgux71rm0wwW1WqjHO394JvltPrp\nrtfrrWTtiHK6mmbIJlPQnxdKQb4wINkMtmjLr/ZysCPrRKaS9bL1z6u5XG2O48W5Wq3Ky8vLtnzM\njqiSwdIK0UQKxAsG8oXBiaZR+HuRcGvphJZ0FZ9O0EnC+tia5Wj9rka+tqAWTR+O5IuEvzfIFwal\n1RRdm6Nr5Ou7m/kI2LeHzPo4+IW02py1+Xwebp7QdIPmd6PJxr5ROnxfkC8MRtYUPUoZ9ES+Preb\nRdT62qX8J+CoW5ketcjXl5GpcEk7QATyhUGpiVdFWot6fb5Xe/S2Ug+1bmUm1qxTmeZ7fc5X63hr\nKQck/H1BvjAoUdrBR7FZA/WeErJWvreUvp4NWsvbWnDzO9iidAPSBeQLgxNFv16mWZVDdkRVFP6x\ntoyM6ne1PWTUqSyLfC3nG23SIOoFA/nCYGSLbZGAo8OnHnye2F5DX0vvlZKnHSzloJsnVLy1nK8J\n2J4/GoaJfAH5wqD43Gwk3myke03M+vyt66hJuka+Jl/rVKaVDlm1g+5eq3VNQ8DfF+QLg+Mj3yi6\nPbTON3qNCF0Q07SDLrZ5+fqRQVHO1yLf7DUBkC+chJ42kSZVPwAzumfN0nUSsXUri2ay+V/17eyv\ntT+DPzTnm20p9tEuDXOgF+QLR6enV0MpZSvZqOWjv2c9e3/8+LGdVqHN0bXKIaow8GVfdm2ThrND\no1u/ky2TLuKFHpAvHJWeKgN7rE1xsubndo4GYk6n072ZbKXs9ms4tFmO5nI1vaBjgKKaXkbBw6Eg\nXzg6tSY5emjkay0iLaXgx/9Y68iHh4cw8vWTKTSPq01yfMOcqFlOJGGVrxevbiUm8oVekC8cnVqD\nHN8sx1IMJlydQhxNJNYhmRb5evlGncq0UY6eo7xuVNFgaQdttEOuF/4E5AsnISsh03s+8o2mEats\n9fADMX2Vg/ZV8FMk9Lq2cy2KfH1j9CjtgIChB+QLRyXK82YNc6LI12Rr6QVLNWgKwv6sVjxkka9J\nUruT6aKZF28mXcv9ZikMIl84FOQLJ6GnYY5OotC0g0nXqht+/PhRptPpVtR22GKdpR10c0UW+VpZ\nmZWYtSodvITtOVW4+picL/SCfOHoZLneqFmORr6a27W5bHZMJpOtqH0NsI98tZRM0wO6bdiO2mKb\nXtufj5qjR83SAVogXzgJUc8G3/5Rc76WVtBpxDaR+OfPn2UymaQ73uzs63w1NWBRrx/1HqUdaumH\nWpMcxAuHgHzhqBzSLCeLfC3PqxOJdSimX7zTw/CdyjTtEInXnyMBLxaLam8G+jXAISBfODo+2vVR\nqh22eNZz3NzcVEvYfJtITQP4jmW+WbpvC5nV8kb9GhAtvBfkC0dFG6Jrjwbfr+Hy8jLt06B/ptYc\nvUWUfvCRsN8qrGVjVDDAKUG+cFQ06lXhap8Ge2zyjcrGWuJtTarIRsJn4q1tmsgqGJAy/AnIF45K\nFPlqiZhe24YJE6/vUuaHYfZGvboA5lMPUQ7YN8np2TSBeOFPQb5wVLx8dVHNFtZMtnd3dzuRb1S3\nq8Mwa+OAPCpgrcOtRb4+7ZBFvogXjgHyhaNiMvSRr26k0A0VumvNR75Wu+sjX32dDI18o9SDyjfq\nUBZFvkgXjgnyhaOike/FxcVO2kG7lWmfhlrawSJfKyNr9QguZXdOmnY3y/K+UeRLrwY4NcgXjkpW\n7aApB98gxzfJyXK+9vz6Wv6ekUW+Udqhp0UkAoZjg3zhqNQW3DTdkI0EihbcfLrBv1507SPfaMEt\naopO5AsfBfKFo6OlZrbgZgLWbcRRe0if81UBl9I3Gy4Sr8/7RpGvitc3ykHAcGyQLxyVP4l8/UaL\nKPK112gRVTto1BvlfFsLbgDHBPlCF7Vf9/WxRax+c4XmfP2IoGiHW1TjeyjRwlsWCUedyYh64ZQg\nX2iS7SSL7mu/3Z5+DbVUw5+IF+Czg3yhSTQSKDureGsSbi20+VwvwLmBfKGL2jggjVQj8daiYBVv\nrcQM4NxAvtDER7fak9c/9mmHSMBRdYNvqKPbigHOEeQLTTTtoI3Q9WzXtYg3euzbTZLzhe8C8oUu\nfIN02zqszdGtQXrURCfK+Wq06w9yvnDuIF9okk2msIg12kbcU/VgJWU+es7qewHOCeQLTaJBmL6e\n1/ft7RkNdHV1FQ7W1HvIF84V5AtdRDnfaEqFRrxZusEOHfeeDcZEvnCuIF9oki24+X69kXBrkfAh\nY4IAzg3kC018ZKoLbtE2Yo2E/dw2/fMXFxfb56+dD8FvC24d2d8DODXIF7rw0a+WmqlQtWTM1+2q\nwPV5o3ON1ky1Q+Sb9XFgdBCcGuQLTfyCW1T14Heq+a3CWYqh9pqemgz1sW+SUzsyATM6CE4N8oUu\nojrfrOIhq9mNFtFarSJrEo4EeWjaIZpWgXjhI0C+0EQjVb+dOEs7ZJFvNAzTXiO69kSC9PcOjXxr\n6Qd9XoBjgnyhCxWvb5auVQ8+53ss8UYyzPK0h0a+tRHxiBdOBfKFJodEvr43byZge17/Or3UcrTv\niXxrlRAApwD5Qhe1Bbco5RD15s0iX3v+6DWVQ6PZllxri20IGE4N8oUmWeTrezxEfXkPEbB/zRaH\niviQagd9DYBTgHyhiqYIfNTbk3bIxNt6vRq1PO97Uwz68eg1AI4N8oUmrTrfKN1w6KJbL7XFtR7B\nHiJlgFOCfKGLmoB95YMX7rEmEftUQXb4UfB26NRiHQ3fs9kC4NggXxgcL7go5/r29rYjULv25/V6\nXZ6fn8uvX7/Kv//+W379+lUeHx/L09NTeX5+LvP5vLy8vJTlcpnKGQnDR4B8YVB8XjU7r1arrSw1\nsl0ul3v3np+fy+/fv3eOp6enMpvNynw+L4vFYk++Kl6NipEunArkC4ORLW5F1xbdmjj17O/NZrPy\n+Pi4jXgfHx/L8/NzeX5+LrPZrLy8vJTFYrGVb5aOIOqFU4J8YVBqlQt6rNfrbaRrqYPsmM1m5fn5\neZtqUPFGaYdaHhj5wqlAvvApaFUuWGphsViUl5eXMp/Py2w22wpVz/6+v6fyfX193RGu5nw17YCA\n4dggXxicTLp6rZGvRrdRhDufz7eSjc6WqrDIN6uiIPKFU4J8YVBqmyX0vFqttnldla8tplmO9/Hx\ncUewtcMW6/xrRaVnAMcG+cLgZJsh9FoX3CyCfXp6Kk9PT+XXr187lQ2z2WwbJVt6wa718evra1mv\n182NGgCnAPnCYHi51XakWc7Xpx0eHx/L79+/y7///rut653P5ztlZLXrbFtx9hjgWCBfGJyeLcCW\n89XI18v3n3/+Kf/880+Zz+d75WPZEb0BRNcAxwb5QjdRROgXyDJxRkcpZW9Tg39s11rTa1UMtthm\n6QYT8MvLS7h1ONtODDAEyBeaWLWBVhwsFotydXW17eFg/RosN6t/ziJVk+Z0Oi3T6bTZo0EF/PT0\nVP7555/y77//lt+/f283TuiONa1caG0RRrowNMgXqmhUa7nS5XK5baCjkyk2m83O4paKdzKZlNls\nViaTyfYoJZ+35sVs/RpscU3la3W7tmEiq1qwzxHgM4B8oYkJTCNfH/H6WtzFYlFubm7KZDIpNzc3\ne9c3NzellLIn2ux6Pp9vpWvlZbZrrVY2FkW+CBg+A8gXmmip1+vr606LyFJ25bxcLsv19XW5ubnZ\nNle3a3/PnrtWY2tnKy2zPK/vUqZbhbONGlQtwGcC+UITlasJ2MTr7+soIbvOzvrctVrbt7e3slgs\n9rYK+y5lfrda6wAYEuQLTTTnqzlevwFiuVzuTbKIplvY2Z6759CGOtpYJ8v5Rs8d/bsAhgL5QhPL\n547H47JarXbuaZ7Xqh90moWe/bU9T5aX1UNzyb6NZJTz9c/jrwGGBvlCE418S/kv4rXqh2hKsV3X\nHttz29kviOk93zDdN1S3ax/51s4AQ4J8ocnb29tOVYOlGnSWm85208PPfdP7RlR/6++1xgjZtb0x\n1J7bXwMMweGTDI8LPwFfAJ027KcPR/f938mOUmIJRvdqZWj+ce15avcBjkjTrcgXAOD4NN06/ojP\nAgAAdkG+AAADgHwBAAYA+QIADADyBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAM+T/ACE/7be2\nhxMvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f83a0e44390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classified as 7 with probability 0.999984\n"
     ]
    }
   ],
   "source": [
    "plt.imshow(val_img[0], cmap='Greys_r')\n",
    "plt.axis('off')\n",
    "plt.show()\n",
    "prob = model.predict(to4d(val_img[0:1]))[0]\n",
    "print 'Classified as %d with probability %f' % (prob.argmax(), max(prob))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also evaluate the accuracy given a data iterator. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 98.650000%\n"
     ]
    }
   ],
   "source": [
    "print 'Validation accuracy: %f%%' % (model.score(val_iter)*100,)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
